1 import pytest                                                          1 import pytest
  2 import sys                                                             2 import sys
  3 import os                                                              3 import os
  4 import tempfile                                                        4 import tempfile
  5 from contextlib import contextmanager                                  5 from contextlib import contextmanager
  6 from warnings import catch_warnings                                    6 from warnings import catch_warnings
  7                                                                        7 
  8 import datetime                                                        8 import datetime
  9 from datetime import timedelta                                         9 from datetime import timedelta
 10 import numpy as np                                                    10 import numpy as np
 11                                                                       11 
 12 import pandas                                                         12 import pandas
 13 import pandas as pd                                                   13 import pandas as pd
 14 from pandas import (Series, DataFrame, Panel, Panel4D, MultiIndex, I 14 from pandas import (Series, DataFrame, Panel, Panel4D, MultiIndex, I
                                                              nt64Index,                                                              nt64Index,
 15                     RangeIndex, Categorical, bdate_range,             15                     RangeIndex, Categorical, bdate_range,
 16                     date_range, timedelta_range, Index, DatetimeInde 16                     date_range, timedelta_range, Index, DatetimeInde
                                                                      x,                                                                      x,
 17                     isnull)                                           17                     isnull)
 18                                                                       18 
 19 from pandas.compat import is_platform_windows, PY3, PY35, text_type   19 from pandas.compat import is_platform_windows, PY3, PY35, text_type
 20 from pandas.io.formats.printing import pprint_thing                   20 from pandas.io.formats.printing import pprint_thing
 21                                                                       21 
 22 tables = pytest.importorskip('tables')                                22 tables = pytest.importorskip('tables')
 23 from pandas.io.pytables import TableIterator                          23 from pandas.io.pytables import TableIterator
 24 from pandas.io.pytables import (HDFStore, get_store, Term, read_hdf,  24 from pandas.io.pytables import (HDFStore, get_store, Term, read_hdf,
 25                                 PossibleDataLossError, ClosedFileErr 25                                 PossibleDataLossError, ClosedFileErr
                                                                     or)                                                                     or)
 26                                                                       26 
 27 from pandas.io import pytables as pytables                            27 from pandas.io import pytables as pytables
 28 import pandas.util.testing as tm                                      28 import pandas.util.testing as tm
 29 from pandas.util.testing import (assert_panel4d_equal,                29 from pandas.util.testing import (assert_panel4d_equal,
 30                                  assert_panel_equal,                  30                                  assert_panel_equal,
 31                                  assert_frame_equal,                  31                                  assert_frame_equal,
 32                                  assert_series_equal,                 32                                  assert_series_equal,
 33                                  set_timezone)                        33                                  set_timezone)
 34 from pandas import concat, Timestamp                                  34 from pandas import concat, Timestamp
 35 from pandas import compat                                             35 from pandas import compat
 36 from pandas.compat import range, lrange, u                            36 from pandas.compat import range, lrange, u
 37 from distutils.version import LooseVersion                            37 from distutils.version import LooseVersion
 38                                                                       38 
 39 _default_compressor = ('blosc' if LooseVersion(tables.__version__) > 39 _default_compressor = ('blosc' if LooseVersion(tables.__version__) >
                                                                 = '2.2'                                                                 = '2.2'
 40                        else 'zlib')                                   40                        else 'zlib')
 41                                                                       41 
 42                                                                       42 
 43 # testing on windows/py3 seems to fault                               43 # testing on windows/py3 seems to fault
 44 # for using compression                                               44 # for using compression
 45 skip_compression = PY3 and is_platform_windows()                      45 skip_compression = PY3 and is_platform_windows()
 46                                                                       46 
 47 # contextmanager to ensure the file cleanup                           47 # contextmanager to ensure the file cleanup
 48                                                                       48 
 49                                                                       49 
 50 def safe_remove(path):                                                50 def safe_remove(path):
 51     if path is not None:                                              51     if path is not None:
 52         try:                                                          52         try:
 53             os.remove(path)                                           53             os.remove(path)
 54         except:                                                       54         except:
 55             pass                                                      55             pass
 56                                                                       56 
 57                                                                       57 
 58 def safe_close(store):                                                58 def safe_close(store):
 59     try:                                                              59     try:
 60         if store is not None:                                         60         if store is not None:
 61             store.close()                                             61             store.close()
 62     except:                                                           62     except:
 63         pass                                                          63         pass
 64                                                                       64 
 65                                                                       65 
 66 def create_tempfile(path):                                            66 def create_tempfile(path):
 67     """ create an unopened named temporary file """                   67     """ create an unopened named temporary file """
 68     return os.path.join(tempfile.gettempdir(), path)                  68     return os.path.join(tempfile.gettempdir(), path)
 69                                                                       69 
 70                                                                       70 
 71 @contextmanager                                                       71 @contextmanager
 72 def ensure_clean_store(path, mode='a', complevel=None, complib=None,  72 def ensure_clean_store(path, mode='a', complevel=None, complib=None,
 73                        fletcher32=False):                             73                        fletcher32=False):
 74                                                                       74 
 75     try:                                                              75     try:
 76                                                                       76 
 77         # put in the temporary path if we don't have one already      77         # put in the temporary path if we don't have one already
 78         if not len(os.path.dirname(path)):                            78         if not len(os.path.dirname(path)):
 79             path = create_tempfile(path)                              79             path = create_tempfile(path)
 80                                                                       80 
 81         store = HDFStore(path, mode=mode, complevel=complevel,        81         store = HDFStore(path, mode=mode, complevel=complevel,
 82                          complib=complib, fletcher32=False)           82                          complib=complib, fletcher32=False)
 83         yield store                                                   83         yield store
 84     finally:                                                          84     finally:
 85         safe_close(store)                                             85         safe_close(store)
 86         if mode == 'w' or mode == 'a':                                86         if mode == 'w' or mode == 'a':
 87             safe_remove(path)                                         87             safe_remove(path)
 88                                                                       88 
 89                                                                       89 
 90 @contextmanager                                                       90 @contextmanager
 91 def ensure_clean_path(path):                                          91 def ensure_clean_path(path):
 92     """                                                               92     """
 93     return essentially a named temporary file that is not opened      93     return essentially a named temporary file that is not opened
 94     and deleted on existing; if path is a list, then create and       94     and deleted on existing; if path is a list, then create and
 95     return list of filenames                                          95     return list of filenames
 96     """                                                               96     """
 97     try:                                                              97     try:
 98         if isinstance(path, list):                                    98         if isinstance(path, list):
 99             filenames = [create_tempfile(p) for p in path]            99             filenames = [create_tempfile(p) for p in path]
 100            yield filenames                                           100            yield filenames
 101        else:                                                         101        else:
 102            filenames = [create_tempfile(path)]                       102            filenames = [create_tempfile(path)]
 103            yield filenames[0]                                        103            yield filenames[0]
 104    finally:                                                          104    finally:
 105        for f in filenames:                                           105        for f in filenames:
 106            safe_remove(f)                                            106            safe_remove(f)
 107                                                                      107
 108                                                                      108
 109# set these parameters so we don't have file sharing                  109# set these parameters so we don't have file sharing
 110tables.parameters.MAX_NUMEXPR_THREADS = 1                             110tables.parameters.MAX_NUMEXPR_THREADS = 1
 111tables.parameters.MAX_BLOSC_THREADS = 1                               111tables.parameters.MAX_BLOSC_THREADS = 1
 112tables.parameters.MAX_THREADS = 1                                     112tables.parameters.MAX_THREADS = 1
 113                                                                      113
 114                                                                      114
 115def _maybe_remove(store, key):                                        115def _maybe_remove(store, key):
 116    """For tests using tables, try removing the table to be sure the 116    """For tests using tables, try removing the table to be sure the
                                                                   re is                                                                   re is
 117    no content from previous tests using the same table name."""      117    no content from previous tests using the same table name."""
 118    try:                                                              118    try:
 119        store.remove(key)                                             119        store.remove(key)
 120    except:                                                           120    except:
 121        pass                                                          121        pass
 122                                                                      122
 123                                                                      123
 124class Base(object):                                                   124class Base(object):
 125                                                                      125
 126    @classmethod                                                      126    @classmethod
 127    def setup_class(cls):                                             127    def setup_class(cls):
 128                                                                      128
 129        # Pytables 3.0.0 deprecates lots of things                    129        # Pytables 3.0.0 deprecates lots of things
 130        tm.reset_testing_mode()                                       130        tm.reset_testing_mode()
 131                                                                      131
 132    @classmethod                                                      132    @classmethod
 133    def teardown_class(cls):                                          133    def teardown_class(cls):
 134                                                                      134
 135        # Pytables 3.0.0 deprecates lots of things                    135        # Pytables 3.0.0 deprecates lots of things
 136        tm.set_testing_mode()                                         136        tm.set_testing_mode()
 137                                                                      137
 138    def setup_method(self, method):                                   138    def setup_method(self, method):
 139        self.path = 'tmp.__%s__.h5' % tm.rands(10)                    139        self.path = 'tmp.__%s__.h5' % tm.rands(10)
 140                                                                      140
 141    def teardown_method(self, method):                                141    def teardown_method(self, method):
 142        pass                                                          142        pass
 143                                                                      143
 144                                                                      144
 145@pytest.mark.single                                                   145@pytest.mark.single
 146class TestHDFStore(Base):                                             146class TestHDFStore(Base):
 147                                                                      147
 148    def test_factory_fun(self):                                       148    def test_factory_fun(self):
 149        path = create_tempfile(self.path)                             149        path = create_tempfile(self.path)
 150        try:                                                          150        try:
 151            with catch_warnings(record=True):                         151            with catch_warnings(record=True):
 152                with get_store(path) as tbl:                          152                with get_store(path) as tbl:
 153                    raise ValueError('blah')                          153                    raise ValueError('blah')
 154        except ValueError:                                            154        except ValueError:
 155            pass                                                      155            pass
 156        finally:                                                      156        finally:
 157            safe_remove(path)                                         157            safe_remove(path)
 158                                                                      158
 159        try:                                                          159        try:
 160            with catch_warnings(record=True):                         160            with catch_warnings(record=True):
 161                with get_store(path) as tbl:                          161                with get_store(path) as tbl:
 162                    tbl['a'] = tm.makeDataFrame()                     162                    tbl['a'] = tm.makeDataFrame()
 163                                                                      163
 164            with catch_warnings(record=True):                         164            with catch_warnings(record=True):
 165                with get_store(path) as tbl:                          165                with get_store(path) as tbl:
 166                    assert len(tbl) == 1                              166                    assert len(tbl) == 1
 167                    assert type(tbl['a']) == DataFrame                167                    assert type(tbl['a']) == DataFrame
 168        finally:                                                      168        finally:
 169            safe_remove(self.path)                                    169            safe_remove(self.path)
 170                                                                      170
 171    def test_context(self):                                           171    def test_context(self):
 172        path = create_tempfile(self.path)                             172        path = create_tempfile(self.path)
 173        try:                                                          173        try:
 174            with HDFStore(path) as tbl:                               174            with HDFStore(path) as tbl:
 175                raise ValueError('blah')                              175                raise ValueError('blah')
 176        except ValueError:                                            176        except ValueError:
 177            pass                                                      177            pass
 178        finally:                                                      178        finally:
 179            safe_remove(path)                                         179            safe_remove(path)
 180                                                                      180
 181        try:                                                          181        try:
 182            with HDFStore(path) as tbl:                               182            with HDFStore(path) as tbl:
 183                tbl['a'] = tm.makeDataFrame()                         183                tbl['a'] = tm.makeDataFrame()
 184                                                                      184
 185            with HDFStore(path) as tbl:                               185            with HDFStore(path) as tbl:
 186                assert len(tbl) == 1                                  186                assert len(tbl) == 1
 187                assert type(tbl['a']) == DataFrame                    187                assert type(tbl['a']) == DataFrame
 188        finally:                                                      188        finally:
 189            safe_remove(path)                                         189            safe_remove(path)
 190                                                                      190
 191    def test_conv_read_write(self):                                   191    def test_conv_read_write(self):
 192        path = create_tempfile(self.path)                             192        path = create_tempfile(self.path)
 193        try:                                                          193        try:
 194            def roundtrip(key, obj, **kwargs):                        194            def roundtrip(key, obj, **kwargs):
 195                obj.to_hdf(path, key, **kwargs)                       195                obj.to_hdf(path, key, **kwargs)
 196                return read_hdf(path, key)                            196                return read_hdf(path, key)
 197                                                                      197
 198            o = tm.makeTimeSeries()                                   198            o = tm.makeTimeSeries()
 199            assert_series_equal(o, roundtrip('series', o))            199            assert_series_equal(o, roundtrip('series', o))
 200                                                                      200
 201            o = tm.makeStringSeries()                                 201            o = tm.makeStringSeries()
 202            assert_series_equal(o, roundtrip('string_series', o))     202            assert_series_equal(o, roundtrip('string_series', o))
 203                                                                      203
 204            o = tm.makeDataFrame()                                    204            o = tm.makeDataFrame()
 205            assert_frame_equal(o, roundtrip('frame', o))              205            assert_frame_equal(o, roundtrip('frame', o))
 206                                                                      206
 207            with catch_warnings(record=True):                         207            with catch_warnings(record=True):
 208                                                                      208
 209                o = tm.makePanel()                                    209                o = tm.makePanel()
 210                assert_panel_equal(o, roundtrip('panel', o))          210                assert_panel_equal(o, roundtrip('panel', o))
 211                                                                      211
 212            # table                                                   212            # table
 213            df = DataFrame(dict(A=lrange(5), B=lrange(5)))            213            df = DataFrame(dict(A=lrange(5), B=lrange(5)))
 214            df.to_hdf(path, 'table', append=True)                     214            df.to_hdf(path, 'table', append=True)
 215            result = read_hdf(path, 'table', where=['index>2'])       215            result = read_hdf(path, 'table', where=['index>2'])
 216            assert_frame_equal(df[df.index > 2], result)              216            assert_frame_equal(df[df.index > 2], result)
 217                                                                      217
 218        finally:                                                      218        finally:
 219            safe_remove(path)                                         219            safe_remove(path)
 220                                                                      220
 221    def test_long_strings(self):                                      221    def test_long_strings(self):
 222                                                                      222
 223        # GH6166                                                      223        # GH6166
 224        # unconversion of long strings was being chopped in earlier   224        # unconversion of long strings was being chopped in earlier
 225        # versions of numpy < 1.7.2                                   225        # versions of numpy < 1.7.2
 226        df = DataFrame({'a': tm.rands_array(100, size=10)},           226        df = DataFrame({'a': tm.rands_array(100, size=10)},
 227                       index=tm.rands_array(100, size=10))            227                       index=tm.rands_array(100, size=10))
 228                                                                      228
 229        with ensure_clean_store(self.path) as store:                  229        with ensure_clean_store(self.path) as store:
 230            store.append('df', df, data_columns=['a'])                230            store.append('df', df, data_columns=['a'])
 231                                                                      231
 232            result = store.select('df')                               232            result = store.select('df')
 233            assert_frame_equal(df, result)                            233            assert_frame_equal(df, result)
 234                                                                      234
 235    def test_api(self):                                               235    def test_api(self):
 236                                                                      236
 237        # GH4584                                                      237        # GH4584
 238        # API issue when to_hdf doesn't acdept append AND format args 238        # API issue when to_hdf doesn't acdept append AND format args
 239        with ensure_clean_path(self.path) as path:                    239        with ensure_clean_path(self.path) as path:
 240                                                                      240
 241            df = tm.makeDataFrame()                                   241            df = tm.makeDataFrame()
 242            df.iloc[:10].to_hdf(path, 'df', append=True, format='tab 242            df.iloc[:10].to_hdf(path, 'df', append=True, format='tab
                                                                    le')                                                                    le')
 243            df.iloc[10:].to_hdf(path, 'df', append=True, format='tab 243            df.iloc[10:].to_hdf(path, 'df', append=True, format='tab
                                                                    le')                                                                    le')
 244            assert_frame_equal(read_hdf(path, 'df'), df)              244            assert_frame_equal(read_hdf(path, 'df'), df)
 245                                                                      245
 246            # append to False                                         246            # append to False
 247            df.iloc[:10].to_hdf(path, 'df', append=False, format='ta 247            df.iloc[:10].to_hdf(path, 'df', append=False, format='ta
                                                                   ble')                                                                   ble')
 248            df.iloc[10:].to_hdf(path, 'df', append=True, format='tab 248            df.iloc[10:].to_hdf(path, 'df', append=True, format='tab
                                                                    le')                                                                    le')
 249            assert_frame_equal(read_hdf(path, 'df'), df)              249            assert_frame_equal(read_hdf(path, 'df'), df)
 250                                                                      250
 251        with ensure_clean_path(self.path) as path:                    251        with ensure_clean_path(self.path) as path:
 252                                                                      252
 253            df = tm.makeDataFrame()                                   253            df = tm.makeDataFrame()
 254            df.iloc[:10].to_hdf(path, 'df', append=True)              254            df.iloc[:10].to_hdf(path, 'df', append=True)
 255            df.iloc[10:].to_hdf(path, 'df', append=True, format='tab 255            df.iloc[10:].to_hdf(path, 'df', append=True, format='tab
                                                                    le')                                                                    le')
 256            assert_frame_equal(read_hdf(path, 'df'), df)              256            assert_frame_equal(read_hdf(path, 'df'), df)
 257                                                                      257
 258            # append to False                                         258            # append to False
 259            df.iloc[:10].to_hdf(path, 'df', append=False, format='ta 259            df.iloc[:10].to_hdf(path, 'df', append=False, format='ta
                                                                   ble')                                                                   ble')
 260            df.iloc[10:].to_hdf(path, 'df', append=True)              260            df.iloc[10:].to_hdf(path, 'df', append=True)
 261            assert_frame_equal(read_hdf(path, 'df'), df)              261            assert_frame_equal(read_hdf(path, 'df'), df)
 262                                                                      262
 263        with ensure_clean_path(self.path) as path:                    263        with ensure_clean_path(self.path) as path:
 264                                                                      264
 265            df = tm.makeDataFrame()                                   265            df = tm.makeDataFrame()
 266            df.to_hdf(path, 'df', append=False, format='fixed')       266            df.to_hdf(path, 'df', append=False, format='fixed')
 267            assert_frame_equal(read_hdf(path, 'df'), df)              267            assert_frame_equal(read_hdf(path, 'df'), df)
 268                                                                      268
 269            df.to_hdf(path, 'df', append=False, format='f')           269            df.to_hdf(path, 'df', append=False, format='f')
 270            assert_frame_equal(read_hdf(path, 'df'), df)              270            assert_frame_equal(read_hdf(path, 'df'), df)
 271                                                                      271
 272            df.to_hdf(path, 'df', append=False)                       272            df.to_hdf(path, 'df', append=False)
 273            assert_frame_equal(read_hdf(path, 'df'), df)              273            assert_frame_equal(read_hdf(path, 'df'), df)
 274                                                                      274
 275            df.to_hdf(path, 'df')                                     275            df.to_hdf(path, 'df')
 276            assert_frame_equal(read_hdf(path, 'df'), df)              276            assert_frame_equal(read_hdf(path, 'df'), df)
 277                                                                      277
 278        with ensure_clean_store(self.path) as store:                  278        with ensure_clean_store(self.path) as store:
 279                                                                      279
 280            path = store._path                                        280            path = store._path
 281            df = tm.makeDataFrame()                                   281            df = tm.makeDataFrame()
 282                                                                      282
 283            _maybe_remove(store, 'df')                                283            _maybe_remove(store, 'df')
 284            store.append('df', df.iloc[:10], append=True, format='ta 284            store.append('df', df.iloc[:10], append=True, format='ta
                                                                   ble')                                                                   ble')
 285            store.append('df', df.iloc[10:], append=True, format='ta 285            store.append('df', df.iloc[10:], append=True, format='ta
                                                                   ble')                                                                   ble')
 286            assert_frame_equal(store.select('df'), df)                286            assert_frame_equal(store.select('df'), df)
 287                                                                      287
 288            # append to False                                         288            # append to False
 289            _maybe_remove(store, 'df')                                289            _maybe_remove(store, 'df')
 290            store.append('df', df.iloc[:10], append=False, format='t 290            store.append('df', df.iloc[:10], append=False, format='t
                                                                  able')                                                                  able')
 291            store.append('df', df.iloc[10:], append=True, format='ta 291            store.append('df', df.iloc[10:], append=True, format='ta
                                                                   ble')                                                                   ble')
 292            assert_frame_equal(store.select('df'), df)                292            assert_frame_equal(store.select('df'), df)
 293                                                                      293
 294            # formats                                                 294            # formats
 295            _maybe_remove(store, 'df')                                295            _maybe_remove(store, 'df')
 296            store.append('df', df.iloc[:10], append=False, format='t 296            store.append('df', df.iloc[:10], append=False, format='t
                                                                  able')                                                                  able')
 297            store.append('df', df.iloc[10:], append=True, format='ta 297            store.append('df', df.iloc[10:], append=True, format='ta
                                                                   ble')                                                                   ble')
 298            assert_frame_equal(store.select('df'), df)                298            assert_frame_equal(store.select('df'), df)
 299                                                                      299
 300            _maybe_remove(store, 'df')                                300            _maybe_remove(store, 'df')
 301            store.append('df', df.iloc[:10], append=False, format='t 301            store.append('df', df.iloc[:10], append=False, format='t
                                                                  able')                                                                  able')
 302            store.append('df', df.iloc[10:], append=True, format=Non 302            store.append('df', df.iloc[10:], append=True, format=Non
                                                                      e)                                                                      e)
 303            assert_frame_equal(store.select('df'), df)                303            assert_frame_equal(store.select('df'), df)
 304                                                                      304
 305        with ensure_clean_path(self.path) as path:                    305        with ensure_clean_path(self.path) as path:
 306                                                                      306
 307            # invalid                                                 307            # invalid
 308            df = tm.makeDataFrame()                                   308            df = tm.makeDataFrame()
 309            pytest.raises(ValueError, df.to_hdf, path,                309            pytest.raises(ValueError, df.to_hdf, path,
 310                          'df', append=True, format='f')              310                          'df', append=True, format='f')
 311            pytest.raises(ValueError, df.to_hdf, path,                311            pytest.raises(ValueError, df.to_hdf, path,
 312                          'df', append=True, format='fixed')          312                          'df', append=True, format='fixed')
 313                                                                      313
 314            pytest.raises(TypeError, df.to_hdf, path,                 314            pytest.raises(TypeError, df.to_hdf, path,
 315                          'df', append=True, format='foo')            315                          'df', append=True, format='foo')
 316            pytest.raises(TypeError, df.to_hdf, path,                 316            pytest.raises(TypeError, df.to_hdf, path,
 317                          'df', append=False, format='bar')           317                          'df', append=False, format='bar')
 318                                                                      318
 319        # File path doesn't exist                                     319        # File path doesn't exist
 320        path = ""                                                     320        path = ""
 321        pytest.raises(compat.FileNotFoundError,                       321        pytest.raises(compat.FileNotFoundError,
 322                      read_hdf, path, 'df')                           322                      read_hdf, path, 'df')
 323                                                                      323
 324    def test_api_default_format(self):                                324    def test_api_default_format(self):
 325                                                                      325
 326        # default_format option                                       326        # default_format option
 327        with ensure_clean_store(self.path) as store:                  327        with ensure_clean_store(self.path) as store:
 328            df = tm.makeDataFrame()                                   328            df = tm.makeDataFrame()
 329                                                                      329
 330            pandas.set_option('io.hdf.default_format', 'fixed')       330            pandas.set_option('io.hdf.default_format', 'fixed')
 331            _maybe_remove(store, 'df')                                331            _maybe_remove(store, 'df')
 332            store.put('df', df)                                       332            store.put('df', df)
 333            assert not store.get_storer('df').is_table                333            assert not store.get_storer('df').is_table
 334            pytest.raises(ValueError, store.append, 'df2', df)        334            pytest.raises(ValueError, store.append, 'df2', df)
 335                                                                      335
 336            pandas.set_option('io.hdf.default_format', 'table')       336            pandas.set_option('io.hdf.default_format', 'table')
 337            _maybe_remove(store, 'df')                                337            _maybe_remove(store, 'df')
 338            store.put('df', df)                                       338            store.put('df', df)
 339            assert store.get_storer('df').is_table                    339            assert store.get_storer('df').is_table
 340            _maybe_remove(store, 'df2')                               340            _maybe_remove(store, 'df2')
 341            store.append('df2', df)                                   341            store.append('df2', df)
 342            assert store.get_storer('df').is_table                    342            assert store.get_storer('df').is_table
 343                                                                      343
 344            pandas.set_option('io.hdf.default_format', None)          344            pandas.set_option('io.hdf.default_format', None)
 345                                                                      345
 346        with ensure_clean_path(self.path) as path:                    346        with ensure_clean_path(self.path) as path:
 347                                                                      347
 348            df = tm.makeDataFrame()                                   348            df = tm.makeDataFrame()
 349                                                                      349
 350            pandas.set_option('io.hdf.default_format', 'fixed')       350            pandas.set_option('io.hdf.default_format', 'fixed')
 351            df.to_hdf(path, 'df')                                     351            df.to_hdf(path, 'df')
 352            with HDFStore(path) as store:                             352            with HDFStore(path) as store:
 353                assert not store.get_storer('df').is_table            353                assert not store.get_storer('df').is_table
 354            pytest.raises(ValueError, df.to_hdf, path, 'df2', append 354            pytest.raises(ValueError, df.to_hdf, path, 'df2', append
                                                                  =True)                                                                  =True)
 355                                                                      355
 356            pandas.set_option('io.hdf.default_format', 'table')       356            pandas.set_option('io.hdf.default_format', 'table')
 357            df.to_hdf(path, 'df3')                                    357            df.to_hdf(path, 'df3')
 358            with HDFStore(path) as store:                             358            with HDFStore(path) as store:
 359                assert store.get_storer('df3').is_table               359                assert store.get_storer('df3').is_table
 360            df.to_hdf(path, 'df4', append=True)                       360            df.to_hdf(path, 'df4', append=True)
 361            with HDFStore(path) as store:                             361            with HDFStore(path) as store:
 362                assert store.get_storer('df4').is_table               362                assert store.get_storer('df4').is_table
 363                                                                      363
 364            pandas.set_option('io.hdf.default_format', None)          364            pandas.set_option('io.hdf.default_format', None)
 365                                                                      365
 366    def test_keys(self):                                              366    def test_keys(self):
 367                                                                      367
 368        with ensure_clean_store(self.path) as store:                  368        with ensure_clean_store(self.path) as store:
 369            store['a'] = tm.makeTimeSeries()                          369            store['a'] = tm.makeTimeSeries()
 370            store['b'] = tm.makeStringSeries()                        370            store['b'] = tm.makeStringSeries()
 371            store['c'] = tm.makeDataFrame()                           371            store['c'] = tm.makeDataFrame()
 372            with catch_warnings(record=True):                         372            with catch_warnings(record=True):
 373                store['d'] = tm.makePanel()                           373                store['d'] = tm.makePanel()
 374                store['foo/bar'] = tm.makePanel()                     374                store['foo/bar'] = tm.makePanel()
 375            assert len(store) == 5                                    375            assert len(store) == 5
 376            expected = set(['/a', '/b', '/c', '/d', '/foo/bar'])      376            expected = set(['/a', '/b', '/c', '/d', '/foo/bar'])
 377            assert set(store.keys()) == expected                      377            assert set(store.keys()) == expected
 378            assert set(store) == expected                             378            assert set(store) == expected
 379                                                                      379
 380    def test_iter_empty(self):                                        380    def test_iter_empty(self):
 381                                                                      381
 382        with ensure_clean_store(self.path) as store:                  382        with ensure_clean_store(self.path) as store:
 383            # GH 12221                                                383            # GH 12221
 384            assert list(store) == []                                  384            assert list(store) == []
 385                                                                      385
 386    def test_repr(self):                                              386    def test_repr(self):
 387                                                                      387
 388        with ensure_clean_store(self.path) as store:                  388        with ensure_clean_store(self.path) as store:
 389            repr(store)                                               389            repr(store)
 390            store['a'] = tm.makeTimeSeries()                          390            store['a'] = tm.makeTimeSeries()
 391            store['b'] = tm.makeStringSeries()                        391            store['b'] = tm.makeStringSeries()
 392            store['c'] = tm.makeDataFrame()                           392            store['c'] = tm.makeDataFrame()
 393                                                                      393
 394            with catch_warnings(record=True):                         394            with catch_warnings(record=True):
 395                store['d'] = tm.makePanel()                           395                store['d'] = tm.makePanel()
 396                store['foo/bar'] = tm.makePanel()                     396                store['foo/bar'] = tm.makePanel()
 397                store.append('e', tm.makePanel())                     397                store.append('e', tm.makePanel())
 398                                                                      398
 399            df = tm.makeDataFrame()                                   399            df = tm.makeDataFrame()
 400            df['obj1'] = 'foo'                                        400            df['obj1'] = 'foo'
 401            df['obj2'] = 'bar'                                        401            df['obj2'] = 'bar'
 402            df['bool1'] = df['A'] > 0                                 402            df['bool1'] = df['A'] > 0
 403            df['bool2'] = df['B'] > 0                                 403            df['bool2'] = df['B'] > 0
 404            df['bool3'] = True                                        404            df['bool3'] = True
 405            df['int1'] = 1                                            405            df['int1'] = 1
 406            df['int2'] = 2                                            406            df['int2'] = 2
 407            df['timestamp1'] = Timestamp('20010102')                  407            df['timestamp1'] = Timestamp('20010102')
 408            df['timestamp2'] = Timestamp('20010103')                  408            df['timestamp2'] = Timestamp('20010103')
 409            df['datetime1'] = datetime.datetime(2001, 1, 2, 0, 0)     409            df['datetime1'] = datetime.datetime(2001, 1, 2, 0, 0)
 410            df['datetime2'] = datetime.datetime(2001, 1, 3, 0, 0)     410            df['datetime2'] = datetime.datetime(2001, 1, 3, 0, 0)
 411            df.loc[3:6, ['obj1']] = np.nan                            411            df.loc[3:6, ['obj1']] = np.nan
 412            df = df._consolidate()._convert(datetime=True)            412            df = df._consolidate()._convert(datetime=True)
 413                                                                      413
 414            # PerformanceWarning                                      414            # PerformanceWarning
 415            with catch_warnings(record=True):                         415            with catch_warnings(record=True):
 416                store['df'] = df                                      416                store['df'] = df
 417                                                                      417
 418            # make a random group in hdf space                        418            # make a random group in hdf space
 419            store._handle.create_group(store._handle.root, 'bah')     419            store._handle.create_group(store._handle.root, 'bah')
 420                                                                      420
 421            repr(store)                                               421            repr(store)
 422            str(store)                                                422            str(store)
 423                                                                      423
 424        # storers                                                     424        # storers
 425        with ensure_clean_store(self.path) as store:                  425        with ensure_clean_store(self.path) as store:
 426                                                                      426
 427            df = tm.makeDataFrame()                                   427            df = tm.makeDataFrame()
 428            store.append('df', df)                                    428            store.append('df', df)
 429                                                                      429
 430            s = store.get_storer('df')                                430            s = store.get_storer('df')
 431            repr(s)                                                   431            repr(s)
 432            str(s)                                                    432            str(s)
 433                                                                      433
 434    def test_contains(self):                                          434    def test_contains(self):
 435                                                                      435
 436        with ensure_clean_store(self.path) as store:                  436        with ensure_clean_store(self.path) as store:
 437            store['a'] = tm.makeTimeSeries()                          437            store['a'] = tm.makeTimeSeries()
 438            store['b'] = tm.makeDataFrame()                           438            store['b'] = tm.makeDataFrame()
 439            store['foo/bar'] = tm.makeDataFrame()                     439            store['foo/bar'] = tm.makeDataFrame()
 440            assert 'a' in store                                       440            assert 'a' in store
 441            assert 'b' in store                                       441            assert 'b' in store
 442            assert 'c' not in store                                   442            assert 'c' not in store
 443            assert 'foo/bar' in store                                 443            assert 'foo/bar' in store
 444            assert '/foo/bar' in store                                444            assert '/foo/bar' in store
 445            assert '/foo/b' not in store                              445            assert '/foo/b' not in store
 446            assert 'bar' not in store                                 446            assert 'bar' not in store
 447                                                                      447
 448            # gh-2694: tables.NaturalNameWarning                      448            # gh-2694: tables.NaturalNameWarning
 449            with catch_warnings(record=True):                         449            with catch_warnings(record=True):
 450                store['node())'] = tm.makeDataFrame()                 450                store['node())'] = tm.makeDataFrame()
 451            assert 'node())' in store                                 451            assert 'node())' in store
 452                                                                      452
 453    def test_versioning(self):                                        453    def test_versioning(self):
 454                                                                      454
 455        with ensure_clean_store(self.path) as store:                  455        with ensure_clean_store(self.path) as store:
 456            store['a'] = tm.makeTimeSeries()                          456            store['a'] = tm.makeTimeSeries()
 457            store['b'] = tm.makeDataFrame()                           457            store['b'] = tm.makeDataFrame()
 458            df = tm.makeTimeDataFrame()                               458            df = tm.makeTimeDataFrame()
 459            _maybe_remove(store, 'df1')                               459            _maybe_remove(store, 'df1')
 460            store.append('df1', df[:10])                              460            store.append('df1', df[:10])
 461            store.append('df1', df[10:])                              461            store.append('df1', df[10:])
 462            assert store.root.a._v_attrs.pandas_version == '0.15.2'   462            assert store.root.a._v_attrs.pandas_version == '0.15.2'
 463            assert store.root.b._v_attrs.pandas_version == '0.15.2'   463            assert store.root.b._v_attrs.pandas_version == '0.15.2'
 464            assert store.root.df1._v_attrs.pandas_version == '0.15.2' 464            assert store.root.df1._v_attrs.pandas_version == '0.15.2'
 465                                                                      465
 466            # write a file and wipe its versioning                    466            # write a file and wipe its versioning
 467            _maybe_remove(store, 'df2')                               467            _maybe_remove(store, 'df2')
 468            store.append('df2', df)                                   468            store.append('df2', df)
 469                                                                      469
 470            # this is an error because its table_type is appendable, 470            # this is an error because its table_type is appendable,
                                                                  but no                                                                  but no
 471            # version info                                            471            # version info
 472            store.get_node('df2')._v_attrs.pandas_version = None      472            store.get_node('df2')._v_attrs.pandas_version = None
 473            pytest.raises(Exception, store.select, 'df2')             473            pytest.raises(Exception, store.select, 'df2')
 474                                                                      474
 475    def test_mode(self):                                              475    def test_mode(self):
 476                                                                      476
 477        df = tm.makeTimeDataFrame()                                   477        df = tm.makeTimeDataFrame()
 478                                                                      478
 479        def check(mode):                                              479        def check(mode):
 480                                                                      480
 481            with ensure_clean_path(self.path) as path:                481            with ensure_clean_path(self.path) as path:
 482                                                                      482
 483                # constructor                                         483                # constructor
 484                if mode in ['r', 'r+']:                               484                if mode in ['r', 'r+']:
 485                    pytest.raises(IOError, HDFStore, path, mode=mode) 485                    pytest.raises(IOError, HDFStore, path, mode=mode)
 486                                                                      486
 487                else:                                                 487                else:
 488                    store = HDFStore(path, mode=mode)                 488                    store = HDFStore(path, mode=mode)
 489                    assert store._handle.mode == mode                 489                    assert store._handle.mode == mode
 490                    store.close()                                     490                    store.close()
 491                                                                      491
 492            with ensure_clean_path(self.path) as path:                492            with ensure_clean_path(self.path) as path:
 493                                                                      493
 494                # context                                             494                # context
 495                if mode in ['r', 'r+']:                               495                if mode in ['r', 'r+']:
 496                    def f():                                          496                    def f():
 497                        with HDFStore(path, mode=mode) as store:  #  497                        with HDFStore(path, mode=mode) as store:  # 
                                                                    noqa                                                                    noqa
 498                            pass                                      498                            pass
 499                    pytest.raises(IOError, f)                         499                    pytest.raises(IOError, f)
 500                else:                                                 500                else:
 501                    with HDFStore(path, mode=mode) as store:          501                    with HDFStore(path, mode=mode) as store:
 502                        assert store._handle.mode == mode             502                        assert store._handle.mode == mode
 503                                                                      503
 504            with ensure_clean_path(self.path) as path:                504            with ensure_clean_path(self.path) as path:
 505                                                                      505
 506                # conv write                                          506                # conv write
 507                if mode in ['r', 'r+']:                               507                if mode in ['r', 'r+']:
 508                    pytest.raises(IOError, df.to_hdf,                 508                    pytest.raises(IOError, df.to_hdf,
 509                                  path, 'df', mode=mode)              509                                  path, 'df', mode=mode)
 510                    df.to_hdf(path, 'df', mode='w')                   510                    df.to_hdf(path, 'df', mode='w')
 511                else:                                                 511                else:
 512                    df.to_hdf(path, 'df', mode=mode)                  512                    df.to_hdf(path, 'df', mode=mode)
 513                                                                      513
 514                # conv read                                           514                # conv read
 515                if mode in ['w']:                                     515                if mode in ['w']:
 516                    pytest.raises(ValueError, read_hdf,               516                    pytest.raises(ValueError, read_hdf,
 517                                  path, 'df', mode=mode)              517                                  path, 'df', mode=mode)
 518                else:                                                 518                else:
 519                    result = read_hdf(path, 'df', mode=mode)          519                    result = read_hdf(path, 'df', mode=mode)
 520                    assert_frame_equal(result, df)                    520                    assert_frame_equal(result, df)
 521                                                                      521
 522        def check_default_mode():                                     522        def check_default_mode():
 523                                                                      523
 524            # read_hdf uses default mode                              524            # read_hdf uses default mode
 525            with ensure_clean_path(self.path) as path:                525            with ensure_clean_path(self.path) as path:
 526                df.to_hdf(path, 'df', mode='w')                       526                df.to_hdf(path, 'df', mode='w')
 527                result = read_hdf(path, 'df')                         527                result = read_hdf(path, 'df')
 528                assert_frame_equal(result, df)                        528                assert_frame_equal(result, df)
 529                                                                      529
 530        check('r')                                                    530        check('r')
 531        check('r+')                                                   531        check('r+')
 532        check('a')                                                    532        check('a')
 533        check('w')                                                    533        check('w')
 534        check_default_mode()                                          534        check_default_mode()
 535                                                                      535
 536    def test_reopen_handle(self):                                     536    def test_reopen_handle(self):
 537                                                                      537
 538        with ensure_clean_path(self.path) as path:                    538        with ensure_clean_path(self.path) as path:
 539                                                                      539
 540            store = HDFStore(path, mode='a')                          540            store = HDFStore(path, mode='a')
 541            store['a'] = tm.makeTimeSeries()                          541            store['a'] = tm.makeTimeSeries()
 542                                                                      542
 543            # invalid mode change                                     543            # invalid mode change
 544            pytest.raises(PossibleDataLossError, store.open, 'w')     544            pytest.raises(PossibleDataLossError, store.open, 'w')
 545            store.close()                                             545            store.close()
 546            assert not store.is_open                                  546            assert not store.is_open
 547                                                                      547
 548            # truncation ok here                                      548            # truncation ok here
 549            store.open('w')                                           549            store.open('w')
 550            assert store.is_open                                      550            assert store.is_open
 551            assert len(store) == 0                                    551            assert len(store) == 0
 552            store.close()                                             552            store.close()
 553            assert not store.is_open                                  553            assert not store.is_open
 554                                                                      554
 555            store = HDFStore(path, mode='a')                          555            store = HDFStore(path, mode='a')
 556            store['a'] = tm.makeTimeSeries()                          556            store['a'] = tm.makeTimeSeries()
 557                                                                      557
 558            # reopen as read                                          558            # reopen as read
 559            store.open('r')                                           559            store.open('r')
 560            assert store.is_open                                      560            assert store.is_open
 561            assert len(store) == 1                                    561            assert len(store) == 1
 562            assert store._mode == 'r'                                 562            assert store._mode == 'r'
 563            store.close()                                             563            store.close()
 564            assert not store.is_open                                  564            assert not store.is_open
 565                                                                      565
 566            # reopen as append                                        566            # reopen as append
 567            store.open('a')                                           567            store.open('a')
 568            assert store.is_open                                      568            assert store.is_open
 569            assert len(store) == 1                                    569            assert len(store) == 1
 570            assert store._mode == 'a'                                 570            assert store._mode == 'a'
 571            store.close()                                             571            store.close()
 572            assert not store.is_open                                  572            assert not store.is_open
 573                                                                      573
 574            # reopen as append (again)                                574            # reopen as append (again)
 575            store.open('a')                                           575            store.open('a')
 576            assert store.is_open                                      576            assert store.is_open
 577            assert len(store) == 1                                    577            assert len(store) == 1
 578            assert store._mode == 'a'                                 578            assert store._mode == 'a'
 579            store.close()                                             579            store.close()
 580            assert not store.is_open                                  580            assert not store.is_open
 581                                                                      581
 582    def test_open_args(self):                                         582    def test_open_args(self):
 583                                                                      583
 584        with ensure_clean_path(self.path) as path:                    584        with ensure_clean_path(self.path) as path:
 585                                                                      585
 586            df = tm.makeDataFrame()                                   586            df = tm.makeDataFrame()
 587                                                                      587
 588            # create an in memory store                               588            # create an in memory store
 589            store = HDFStore(path, mode='a', driver='H5FD_CORE',      589            store = HDFStore(path, mode='a', driver='H5FD_CORE',
 590                             driver_core_backing_store=0)             590                             driver_core_backing_store=0)
 591            store['df'] = df                                          591            store['df'] = df
 592            store.append('df2', df)                                   592            store.append('df2', df)
 593                                                                      593
 594            tm.assert_frame_equal(store['df'], df)                    594            tm.assert_frame_equal(store['df'], df)
 595            tm.assert_frame_equal(store['df2'], df)                   595            tm.assert_frame_equal(store['df2'], df)
 596                                                                      596
 597            store.close()                                             597            store.close()
 598                                                                      598
 599            # the file should not have actually been written          599            # the file should not have actually been written
 600            assert not os.path.exists(path)                           600            assert not os.path.exists(path)
 601                                                                      601
 602    def test_flush(self):                                             602    def test_flush(self):
 603                                                                      603
 604        with ensure_clean_store(self.path) as store:                  604        with ensure_clean_store(self.path) as store:
 605            store['a'] = tm.makeTimeSeries()                          605            store['a'] = tm.makeTimeSeries()
 606            store.flush()                                             606            store.flush()
 607            store.flush(fsync=True)                                   607            store.flush(fsync=True)
 608                                                                      608
 609    def test_get(self):                                               609    def test_get(self):
 610                                                                      610
 611        with ensure_clean_store(self.path) as store:                  611        with ensure_clean_store(self.path) as store:
 612            store['a'] = tm.makeTimeSeries()                          612            store['a'] = tm.makeTimeSeries()
 613            left = store.get('a')                                     613            left = store.get('a')
 614            right = store['a']                                        614            right = store['a']
 615            tm.assert_series_equal(left, right)                       615            tm.assert_series_equal(left, right)
 616                                                                      616
 617            left = store.get('/a')                                    617            left = store.get('/a')
 618            right = store['/a']                                       618            right = store['/a']
 619            tm.assert_series_equal(left, right)                       619            tm.assert_series_equal(left, right)
 620                                                                      620
 621            pytest.raises(KeyError, store.get, 'b')                   621            pytest.raises(KeyError, store.get, 'b')
 622                                                                      622
 623    def test_getattr(self):                                           623    def test_getattr(self):
 624                                                                      624
 625        with ensure_clean_store(self.path) as store:                  625        with ensure_clean_store(self.path) as store:
 626                                                                      626
 627            s = tm.makeTimeSeries()                                   627            s = tm.makeTimeSeries()
 628            store['a'] = s                                            628            store['a'] = s
 629                                                                      629
 630            # test attribute access                                   630            # test attribute access
 631            result = store.a                                          631            result = store.a
 632            tm.assert_series_equal(result, s)                         632            tm.assert_series_equal(result, s)
 633            result = getattr(store, 'a')                              633            result = getattr(store, 'a')
 634            tm.assert_series_equal(result, s)                         634            tm.assert_series_equal(result, s)
 635                                                                      635
 636            df = tm.makeTimeDataFrame()                               636            df = tm.makeTimeDataFrame()
 637            store['df'] = df                                          637            store['df'] = df
 638            result = store.df                                         638            result = store.df
 639            tm.assert_frame_equal(result, df)                         639            tm.assert_frame_equal(result, df)
 640                                                                      640
 641            # errors                                                  641            # errors
 642            pytest.raises(AttributeError, getattr, store, 'd')        642            pytest.raises(AttributeError, getattr, store, 'd')
 643                                                                      643
 644            for x in ['mode', 'path', 'handle', 'complib']:           644            for x in ['mode', 'path', 'handle', 'complib']:
 645                pytest.raises(AttributeError, getattr, store, x)      645                pytest.raises(AttributeError, getattr, store, x)
 646                                                                      646
 647            # not stores                                              647            # not stores
 648            for x in ['mode', 'path', 'handle', 'complib']:           648            for x in ['mode', 'path', 'handle', 'complib']:
 649                getattr(store, "_%s" % x)                             649                getattr(store, "_%s" % x)
 650                                                                      650
 651    def test_put(self):                                               651    def test_put(self):
 652                                                                      652
 653        with ensure_clean_store(self.path) as store:                  653        with ensure_clean_store(self.path) as store:
 654                                                                      654
 655            ts = tm.makeTimeSeries()                                  655            ts = tm.makeTimeSeries()
 656            df = tm.makeTimeDataFrame()                               656            df = tm.makeTimeDataFrame()
 657            store['a'] = ts                                           657            store['a'] = ts
 658            store['b'] = df[:10]                                      658            store['b'] = df[:10]
 659            store['foo/bar/bah'] = df[:10]                            659            store['foo/bar/bah'] = df[:10]
 660            store['foo'] = df[:10]                                    660            store['foo'] = df[:10]
 661            store['/foo'] = df[:10]                                   661            store['/foo'] = df[:10]
 662            store.put('c', df[:10], format='table')                   662            store.put('c', df[:10], format='table')
 663                                                                      663
 664            # not OK, not a table                                     664            # not OK, not a table
 665            pytest.raises(                                            665            pytest.raises(
 666                ValueError, store.put, 'b', df[10:], append=True)     666                ValueError, store.put, 'b', df[10:], append=True)
 667                                                                      667
 668            # node does not currently exist, test _is_table_type ret 668            # node does not currently exist, test _is_table_type ret
                                                              urns False                                                              urns False
 669            # in this case                                            669            # in this case
 670            # _maybe_remove(store, 'f')                               670            # _maybe_remove(store, 'f')
 671            # pytest.raises(ValueError, store.put, 'f', df[10:],      671            # pytest.raises(ValueError, store.put, 'f', df[10:],
 672            #                   append=True)                          672            #                   append=True)
 673                                                                      673
 674            # can't put to a table (use append instead)               674            # can't put to a table (use append instead)
 675            pytest.raises(ValueError, store.put, 'c', df[10:], appen 675            pytest.raises(ValueError, store.put, 'c', df[10:], appen
                                                                 d=True)                                                                 d=True)
 676                                                                      676
 677            # overwrite table                                         677            # overwrite table
 678            store.put('c', df[:10], format='table', append=False)     678            store.put('c', df[:10], format='table', append=False)
 679            tm.assert_frame_equal(df[:10], store['c'])                679            tm.assert_frame_equal(df[:10], store['c'])
 680                                                                      680
 681    def test_put_string_index(self):                                  681    def test_put_string_index(self):
 682                                                                      682
 683        with ensure_clean_store(self.path) as store:                  683        with ensure_clean_store(self.path) as store:
 684                                                                      684
 685            index = Index(                                            685            index = Index(
 686                ["I am a very long string index: %s" % i for i in ra 686                ["I am a very long string index: %s" % i for i in ra
                                                               nge(20)])                                                               nge(20)])
 687            s = Series(np.arange(20), index=index)                    687            s = Series(np.arange(20), index=index)
 688            df = DataFrame({'A': s, 'B': s})                          688            df = DataFrame({'A': s, 'B': s})
 689                                                                      689
 690            store['a'] = s                                            690            store['a'] = s
 691            tm.assert_series_equal(store['a'], s)                     691            tm.assert_series_equal(store['a'], s)
 692                                                                      692
 693            store['b'] = df                                           693            store['b'] = df
 694            tm.assert_frame_equal(store['b'], df)                     694            tm.assert_frame_equal(store['b'], df)
 695                                                                      695
 696            # mixed length                                            696            # mixed length
 697            index = Index(['abcdefghijklmnopqrstuvwxyz1234567890'] +  697            index = Index(['abcdefghijklmnopqrstuvwxyz1234567890'] +
 698                          ["I am a very long string index: %s" % i    698                          ["I am a very long string index: %s" % i
 699                           for i in range(20)])                       699                           for i in range(20)])
 700            s = Series(np.arange(21), index=index)                    700            s = Series(np.arange(21), index=index)
 701            df = DataFrame({'A': s, 'B': s})                          701            df = DataFrame({'A': s, 'B': s})
 702            store['a'] = s                                            702            store['a'] = s
 703            tm.assert_series_equal(store['a'], s)                     703            tm.assert_series_equal(store['a'], s)
 704                                                                      704
 705            store['b'] = df                                           705            store['b'] = df
 706            tm.assert_frame_equal(store['b'], df)                     706            tm.assert_frame_equal(store['b'], df)
 707                                                                      707
 708    def test_put_compression(self):                                   708    def test_put_compression(self):
 709                                                                      709
 710        with ensure_clean_store(self.path) as store:                  710        with ensure_clean_store(self.path) as store:
 711            df = tm.makeTimeDataFrame()                               711            df = tm.makeTimeDataFrame()
 712                                                                      712
 713            store.put('c', df, format='table', complib='zlib')        713            store.put('c', df, format='table', complib='zlib')
 714            tm.assert_frame_equal(store['c'], df)                     714            tm.assert_frame_equal(store['c'], df)
 715                                                                      715
 716            # can't compress if format='fixed'                        716            # can't compress if format='fixed'
 717            pytest.raises(ValueError, store.put, 'b', df,             717            pytest.raises(ValueError, store.put, 'b', df,
 718                          format='fixed', complib='zlib')             718                          format='fixed', complib='zlib')
 719                                                                      719
 720    def test_put_compression_blosc(self):                             720    def test_put_compression_blosc(self):
 721        tm.skip_if_no_package('tables', min_version='2.2',            721        tm.skip_if_no_package('tables', min_version='2.2',
 722                              app='blosc support')                    722                              app='blosc support')
 723        if skip_compression:                                          723        if skip_compression:
 724            pytest.skip("skipping on windows/PY3")                    724            pytest.skip("skipping on windows/PY3")
 725                                                                      725
 726        df = tm.makeTimeDataFrame()                                   726        df = tm.makeTimeDataFrame()
 727                                                                      727
 728        with ensure_clean_store(self.path) as store:                  728        with ensure_clean_store(self.path) as store:
 729                                                                      729
 730            # can't compress if format='fixed'                        730            # can't compress if format='fixed'
 731            pytest.raises(ValueError, store.put, 'b', df,             731            pytest.raises(ValueError, store.put, 'b', df,
 732                          format='fixed', complib='blosc')            732                          format='fixed', complib='blosc')
 733                                                                      733
 734            store.put('c', df, format='table', complib='blosc')       734            store.put('c', df, format='table', complib='blosc')
 735            tm.assert_frame_equal(store['c'], df)                     735            tm.assert_frame_equal(store['c'], df)
 736                                                                      736
 737    def test_complibs(self):                                          737    def test_complibs(self):
 738        # GH14478                                                     738        # GH14478
 739        df = tm.makeDataFrame()                                       739        df = tm.makeDataFrame()
 740                                                                      740
 741        # Building list of all complibs and complevels tuples         741        # Building list of all complibs and complevels tuples
 742        all_complibs = tables.filters.all_complibs                    742        all_complibs = tables.filters.all_complibs
 743        # Remove lzo if its not available on this platform            743        # Remove lzo if its not available on this platform
 744        if not tables.which_lib_version('lzo'):                       744        if not tables.which_lib_version('lzo'):
 745            all_complibs.remove('lzo')                                745            all_complibs.remove('lzo')
 746        all_levels = range(0, 10)                                     746        all_levels = range(0, 10)
 747        all_tests = [(lib, lvl) for lib in all_complibs for lvl in a 747        all_tests = [(lib, lvl) for lib in all_complibs for lvl in a
                                                              ll_levels]                                                              ll_levels]
 748                                                                      748
 749        for (lib, lvl) in all_tests:                                  749        for (lib, lvl) in all_tests:
 750            with ensure_clean_path(self.path) as tmpfile:             750            with ensure_clean_path(self.path) as tmpfile:
 751                gname = 'foo'                                         751                gname = 'foo'
 752                                                                      752
 753                # Write and read file to see if data is consistent    753                # Write and read file to see if data is consistent
 754                df.to_hdf(tmpfile, gname, complib=lib, complevel=lvl) 754                df.to_hdf(tmpfile, gname, complib=lib, complevel=lvl)
 755                result = pd.read_hdf(tmpfile, gname)                  755                result = pd.read_hdf(tmpfile, gname)
 756                tm.assert_frame_equal(result, df)                     756                tm.assert_frame_equal(result, df)
 757                                                                      757
 758                # Open file and check metadata                        758                # Open file and check metadata
 759                # for correct amount of compression                   759                # for correct amount of compression
 760                h5table = tables.open_file(tmpfile, mode='r')         760                h5table = tables.open_file(tmpfile, mode='r')
 761                for node in h5table.walk_nodes(where='/' + gname,     761                for node in h5table.walk_nodes(where='/' + gname,
 762                                               classname='Leaf'):     762                                               classname='Leaf'):
 763                    assert node.filters.complevel == lvl              763                    assert node.filters.complevel == lvl
 764                    if lvl == 0:                                      764                    if lvl == 0:
 765                        assert node.filters.complib is None           765                        assert node.filters.complib is None
 766                    else:                                             766                    else:
 767                        assert node.filters.complib == lib            767                        assert node.filters.complib == lib
 768                h5table.close()                                       768                h5table.close()
 769                                                                      769
 770    def test_put_integer(self):                                       770    def test_put_integer(self):
 771        # non-date, non-string index                                  771        # non-date, non-string index
 772        df = DataFrame(np.random.randn(50, 100))                      772        df = DataFrame(np.random.randn(50, 100))
 773        self._check_roundtrip(df, tm.assert_frame_equal)              773        self._check_roundtrip(df, tm.assert_frame_equal)
 774                                                                      774
 775    def test_put_mixed_type(self):                                    775    def test_put_mixed_type(self):
 776        df = tm.makeTimeDataFrame()                                   776        df = tm.makeTimeDataFrame()
 777        df['obj1'] = 'foo'                                            777        df['obj1'] = 'foo'
 778        df['obj2'] = 'bar'                                            778        df['obj2'] = 'bar'
 779        df['bool1'] = df['A'] > 0                                     779        df['bool1'] = df['A'] > 0
 780        df['bool2'] = df['B'] > 0                                     780        df['bool2'] = df['B'] > 0
 781        df['bool3'] = True                                            781        df['bool3'] = True
 782        df['int1'] = 1                                                782        df['int1'] = 1
 783        df['int2'] = 2                                                783        df['int2'] = 2
 784        df['timestamp1'] = Timestamp('20010102')                      784        df['timestamp1'] = Timestamp('20010102')
 785        df['timestamp2'] = Timestamp('20010103')                      785        df['timestamp2'] = Timestamp('20010103')
 786        df['datetime1'] = datetime.datetime(2001, 1, 2, 0, 0)         786        df['datetime1'] = datetime.datetime(2001, 1, 2, 0, 0)
 787        df['datetime2'] = datetime.datetime(2001, 1, 3, 0, 0)         787        df['datetime2'] = datetime.datetime(2001, 1, 3, 0, 0)
 788        df.loc[3:6, ['obj1']] = np.nan                                788        df.loc[3:6, ['obj1']] = np.nan
 789        df = df._consolidate()._convert(datetime=True)                789        df = df._consolidate()._convert(datetime=True)
 790                                                                      790
 791        with ensure_clean_store(self.path) as store:                  791        with ensure_clean_store(self.path) as store:
 792            _maybe_remove(store, 'df')                                792            _maybe_remove(store, 'df')
 793                                                                      793
 794            # PerformanceWarning                                      794            # PerformanceWarning
 795            with catch_warnings(record=True):                         795            with catch_warnings(record=True):
 796                store.put('df', df)                                   796                store.put('df', df)
 797                                                                      797
 798            expected = store.get('df')                                798            expected = store.get('df')
 799            tm.assert_frame_equal(expected, df)                       799            tm.assert_frame_equal(expected, df)
 800                                                                      800
 801    def test_append(self):                                            801    def test_append(self):
 802                                                                      802
 803        with ensure_clean_store(self.path) as store:                  803        with ensure_clean_store(self.path) as store:
 804                                                                      804
 805            # this is allowed by almost always don't want to do it    805            # this is allowed by almost always don't want to do it
 806            # tables.NaturalNameWarning):                             806            # tables.NaturalNameWarning):
 807            with catch_warnings(record=True):                         807            with catch_warnings(record=True):
 808                                                                      808
 809                df = tm.makeTimeDataFrame()                           809                df = tm.makeTimeDataFrame()
 810                _maybe_remove(store, 'df1')                           810                _maybe_remove(store, 'df1')
 811                store.append('df1', df[:10])                          811                store.append('df1', df[:10])
 812                store.append('df1', df[10:])                          812                store.append('df1', df[10:])
 813                tm.assert_frame_equal(store['df1'], df)               813                tm.assert_frame_equal(store['df1'], df)
 814                                                                      814
 815                _maybe_remove(store, 'df2')                           815                _maybe_remove(store, 'df2')
 816                store.put('df2', df[:10], format='table')             816                store.put('df2', df[:10], format='table')
 817                store.append('df2', df[10:])                          817                store.append('df2', df[10:])
 818                tm.assert_frame_equal(store['df2'], df)               818                tm.assert_frame_equal(store['df2'], df)
 819                                                                      819
 820                _maybe_remove(store, 'df3')                           820                _maybe_remove(store, 'df3')
 821                store.append('/df3', df[:10])                         821                store.append('/df3', df[:10])
 822                store.append('/df3', df[10:])                         822                store.append('/df3', df[10:])
 823                tm.assert_frame_equal(store['df3'], df)               823                tm.assert_frame_equal(store['df3'], df)
 824                                                                      824
 825                # this is allowed by almost always don't want to do  825                # this is allowed by almost always don't want to do 
                                                                      it                                                                      it
 826                # tables.NaturalNameWarning                           826                # tables.NaturalNameWarning
 827                _maybe_remove(store, '/df3 foo')                      827                _maybe_remove(store, '/df3 foo')
 828                store.append('/df3 foo', df[:10])                     828                store.append('/df3 foo', df[:10])
 829                store.append('/df3 foo', df[10:])                     829                store.append('/df3 foo', df[10:])
 830                tm.assert_frame_equal(store['df3 foo'], df)           830                tm.assert_frame_equal(store['df3 foo'], df)
 831                                                                      831
 832                # panel                                               832                # panel
 833                wp = tm.makePanel()                                   833                wp = tm.makePanel()
 834                _maybe_remove(store, 'wp1')                           834                _maybe_remove(store, 'wp1')
 835                store.append('wp1', wp.iloc[:, :10, :])               835                store.append('wp1', wp.iloc[:, :10, :])
 836                store.append('wp1', wp.iloc[:, 10:, :])               836                store.append('wp1', wp.iloc[:, 10:, :])
 837                assert_panel_equal(store['wp1'], wp)                  837                assert_panel_equal(store['wp1'], wp)
 838                                                                      838
 839                # ndim                                                839                # ndim
 840                p4d = tm.makePanel4D()                                840                p4d = tm.makePanel4D()
 841                _maybe_remove(store, 'p4d')                           841                _maybe_remove(store, 'p4d')
 842                store.append('p4d', p4d.iloc[:, :, :10, :])           842                store.append('p4d', p4d.iloc[:, :, :10, :])
 843                store.append('p4d', p4d.iloc[:, :, 10:, :])           843                store.append('p4d', p4d.iloc[:, :, 10:, :])
 844                assert_panel4d_equal(store['p4d'], p4d)               844                assert_panel4d_equal(store['p4d'], p4d)
 845                                                                      845
 846                # test using axis labels                              846                # test using axis labels
 847                _maybe_remove(store, 'p4d')                           847                _maybe_remove(store, 'p4d')
 848                store.append('p4d', p4d.iloc[:, :, :10, :], axes=[    848                store.append('p4d', p4d.iloc[:, :, :10, :], axes=[
 849                    'items', 'major_axis', 'minor_axis'])             849                    'items', 'major_axis', 'minor_axis'])
 850                store.append('p4d', p4d.iloc[:, :, 10:, :], axes=[    850                store.append('p4d', p4d.iloc[:, :, 10:, :], axes=[
 851                    'items', 'major_axis', 'minor_axis'])             851                    'items', 'major_axis', 'minor_axis'])
 852                assert_panel4d_equal(store['p4d'], p4d)               852                assert_panel4d_equal(store['p4d'], p4d)
 853                                                                      853
 854                # test using differnt number of items on each axis    854                # test using differnt number of items on each axis
 855                p4d2 = p4d.copy()                                     855                p4d2 = p4d.copy()
 856                p4d2['l4'] = p4d['l1']                                856                p4d2['l4'] = p4d['l1']
 857                p4d2['l5'] = p4d['l1']                                857                p4d2['l5'] = p4d['l1']
 858                _maybe_remove(store, 'p4d2')                          858                _maybe_remove(store, 'p4d2')
 859                store.append(                                         859                store.append(
 860                    'p4d2', p4d2, axes=['items', 'major_axis', 'mino 860                    'p4d2', p4d2, axes=['items', 'major_axis', 'mino
                                                               r_axis'])                                                               r_axis'])
 861                assert_panel4d_equal(store['p4d2'], p4d2)             861                assert_panel4d_equal(store['p4d2'], p4d2)
 862                                                                      862
 863                # test using differt order of items on the non-index 863                # test using differt order of items on the non-index
                                                                    axes                                                                    axes
 864                _maybe_remove(store, 'wp1')                           864                _maybe_remove(store, 'wp1')
 865                wp_append1 = wp.iloc[:, :10, :]                       865                wp_append1 = wp.iloc[:, :10, :]
 866                store.append('wp1', wp_append1)                       866                store.append('wp1', wp_append1)
 867                wp_append2 = wp.iloc[:, 10:, :].reindex(items=wp.ite 867                wp_append2 = wp.iloc[:, 10:, :].reindex(items=wp.ite
                                                               ms[::-1])                                                               ms[::-1])
 868                store.append('wp1', wp_append2)                       868                store.append('wp1', wp_append2)
 869                assert_panel_equal(store['wp1'], wp)                  869                assert_panel_equal(store['wp1'], wp)
 870                                                                      870
 871                # dtype issues - mizxed type in a single object colu 871                # dtype issues - mizxed type in a single object colu
                                                                      mn                                                                      mn
 872                df = DataFrame(data=[[1, 2], [0, 1], [1, 2], [0, 0]]) 872                df = DataFrame(data=[[1, 2], [0, 1], [1, 2], [0, 0]])
 873                df['mixed_column'] = 'testing'                        873                df['mixed_column'] = 'testing'
 874                df.loc[2, 'mixed_column'] = np.nan                    874                df.loc[2, 'mixed_column'] = np.nan
 875                _maybe_remove(store, 'df')                            875                _maybe_remove(store, 'df')
 876                store.append('df', df)                                876                store.append('df', df)
 877                tm.assert_frame_equal(store['df'], df)                877                tm.assert_frame_equal(store['df'], df)
 878                                                                      878
 879                # uints - test storage of uints                       879                # uints - test storage of uints
 880                uint_data = DataFrame({                               880                uint_data = DataFrame({
 881                    'u08': Series(np.random.randint(0, high=255, siz 881                    'u08': Series(np.random.randint(0, high=255, siz
                                                                   e=5),                                                                   e=5),
 882                                  dtype=np.uint8),                    882                                  dtype=np.uint8),
 883                    'u16': Series(np.random.randint(0, high=65535, s 883                    'u16': Series(np.random.randint(0, high=65535, s
                                                                 ize=5),                                                                 ize=5),
 884                                  dtype=np.uint16),                   884                                  dtype=np.uint16),
 885                    'u32': Series(np.random.randint(0, high=2**30, s 885                    'u32': Series(np.random.randint(0, high=2**30, s
                                                                 ize=5),                                                                 ize=5),
 886                                  dtype=np.uint32),                   886                                  dtype=np.uint32),
 887                    'u64': Series([2**58, 2**59, 2**60, 2**61, 2**62 887                    'u64': Series([2**58, 2**59, 2**60, 2**61, 2**62
                                                                      ],                                                                      ],
 888                                  dtype=np.uint64)}, index=np.arange 888                                  dtype=np.uint64)}, index=np.arange
                                                                    (5))                                                                    (5))
 889                _maybe_remove(store, 'uints')                         889                _maybe_remove(store, 'uints')
 890                store.append('uints', uint_data)                      890                store.append('uints', uint_data)
 891                tm.assert_frame_equal(store['uints'], uint_data)      891                tm.assert_frame_equal(store['uints'], uint_data)
 892                                                                      892
 893                # uints - test storage of uints in indexable columns  893                # uints - test storage of uints in indexable columns
 894                _maybe_remove(store, 'uints')                         894                _maybe_remove(store, 'uints')
 895                # 64-bit indices not yet supported                    895                # 64-bit indices not yet supported
 896                store.append('uints', uint_data, data_columns=[       896                store.append('uints', uint_data, data_columns=[
 897                             'u08', 'u16', 'u32'])                    897                             'u08', 'u16', 'u32'])
 898                tm.assert_frame_equal(store['uints'], uint_data)      898                tm.assert_frame_equal(store['uints'], uint_data)
 899                                                                      899
 900    def test_append_series(self):                                     900    def test_append_series(self):
 901                                                                      901
 902        with ensure_clean_store(self.path) as store:                  902        with ensure_clean_store(self.path) as store:
 903                                                                      903
 904            # basic                                                   904            # basic
 905            ss = tm.makeStringSeries()                                905            ss = tm.makeStringSeries()
 906            ts = tm.makeTimeSeries()                                  906            ts = tm.makeTimeSeries()
 907            ns = Series(np.arange(100))                               907            ns = Series(np.arange(100))
 908                                                                      908
 909            store.append('ss', ss)                                    909            store.append('ss', ss)
 910            result = store['ss']                                      910            result = store['ss']
 911            tm.assert_series_equal(result, ss)                        911            tm.assert_series_equal(result, ss)
 912            assert result.name is None                                912            assert result.name is None
 913                                                                      913
 914            store.append('ts', ts)                                    914            store.append('ts', ts)
 915            result = store['ts']                                      915            result = store['ts']
 916            tm.assert_series_equal(result, ts)                        916            tm.assert_series_equal(result, ts)
 917            assert result.name is None                                917            assert result.name is None
 918                                                                      918
 919            ns.name = 'foo'                                           919            ns.name = 'foo'
 920            store.append('ns', ns)                                    920            store.append('ns', ns)
 921            result = store['ns']                                      921            result = store['ns']
 922            tm.assert_series_equal(result, ns)                        922            tm.assert_series_equal(result, ns)
 923            assert result.name == ns.name                             923            assert result.name == ns.name
 924                                                                      924
 925            # select on the values                                    925            # select on the values
 926            expected = ns[ns > 60]                                    926            expected = ns[ns > 60]
 927            result = store.select('ns', 'foo>60')                     927            result = store.select('ns', 'foo>60')
 928            tm.assert_series_equal(result, expected)                  928            tm.assert_series_equal(result, expected)
 929                                                                      929
 930            # select on the index and values                          930            # select on the index and values
 931            expected = ns[(ns > 70) & (ns.index < 90)]                931            expected = ns[(ns > 70) & (ns.index < 90)]
 932            result = store.select('ns', 'foo>70 and index<90')        932            result = store.select('ns', 'foo>70 and index<90')
 933            tm.assert_series_equal(result, expected)                  933            tm.assert_series_equal(result, expected)
 934                                                                      934
 935            # multi-index                                             935            # multi-index
 936            mi = DataFrame(np.random.randn(5, 1), columns=['A'])      936            mi = DataFrame(np.random.randn(5, 1), columns=['A'])
 937            mi['B'] = np.arange(len(mi))                              937            mi['B'] = np.arange(len(mi))
 938            mi['C'] = 'foo'                                           938            mi['C'] = 'foo'
 939            mi.loc[3:5, 'C'] = 'bar'                                  939            mi.loc[3:5, 'C'] = 'bar'
 940            mi.set_index(['C', 'B'], inplace=True)                    940            mi.set_index(['C', 'B'], inplace=True)
 941            s = mi.stack()                                            941            s = mi.stack()
 942            s.index = s.index.droplevel(2)                            942            s.index = s.index.droplevel(2)
 943            store.append('mi', s)                                     943            store.append('mi', s)
 944            tm.assert_series_equal(store['mi'], s)                    944            tm.assert_series_equal(store['mi'], s)
 945                                                                      945
 946    def test_store_index_types(self):                                 946    def test_store_index_types(self):
 947        # GH5386                                                      947        # GH5386
 948        # test storing various index types                            948        # test storing various index types
 949                                                                      949
 950        with ensure_clean_store(self.path) as store:                  950        with ensure_clean_store(self.path) as store:
 951                                                                      951
 952            def check(format, index):                                 952            def check(format, index):
 953                df = DataFrame(np.random.randn(10, 2), columns=list( 953                df = DataFrame(np.random.randn(10, 2), columns=list(
                                                                  'AB'))                                                                  'AB'))
 954                df.index = index(len(df))                             954                df.index = index(len(df))
 955                                                                      955
 956                _maybe_remove(store, 'df')                            956                _maybe_remove(store, 'df')
 957                store.put('df', df, format=format)                    957                store.put('df', df, format=format)
 958                assert_frame_equal(df, store['df'])                   958                assert_frame_equal(df, store['df'])
 959                                                                      959
 960            for index in [tm.makeFloatIndex, tm.makeStringIndex,      960            for index in [tm.makeFloatIndex, tm.makeStringIndex,
 961                          tm.makeIntIndex, tm.makeDateIndex]:         961                          tm.makeIntIndex, tm.makeDateIndex]:
 962                                                                      962
 963                check('table', index)                                 963                check('table', index)
 964                check('fixed', index)                                 964                check('fixed', index)
 965                                                                      965
 966            # period index currently broken for table                 966            # period index currently broken for table
 967            # seee GH7796 FIXME                                       967            # seee GH7796 FIXME
 968            check('fixed', tm.makePeriodIndex)                        968            check('fixed', tm.makePeriodIndex)
 969            # check('table',tm.makePeriodIndex)                       969            # check('table',tm.makePeriodIndex)
 970                                                                      970
 971            # unicode                                                 971            # unicode
 972            index = tm.makeUnicodeIndex                               972            index = tm.makeUnicodeIndex
 973            if compat.PY3:                                            973            if compat.PY3:
 974                check('table', index)                                 974                check('table', index)
 975                check('fixed', index)                                 975                check('fixed', index)
 976            else:                                                     976            else:
 977                                                                      977
 978                # only support for fixed types (and they have a perf 978                # only support for fixed types (and they have a perf
                                                                warning)                                                                warning)
 979                pytest.raises(TypeError, check, 'table', index)       979                pytest.raises(TypeError, check, 'table', index)
 980                                                                      980
 981                # PerformanceWarning                                  981                # PerformanceWarning
 982                with catch_warnings(record=True):                     982                with catch_warnings(record=True):
 983                    check('fixed', index)                             983                    check('fixed', index)
 984                                                                      984
 985    def test_encoding(self):                                          985    def test_encoding(self):
 986                                                                      986
 987        if sys.byteorder != 'little':                                 987        if sys.byteorder != 'little':
 988            pytest.skip('system byteorder is not little')             988            pytest.skip('system byteorder is not little')
 989                                                                      989
 990        with ensure_clean_store(self.path) as store:                  990        with ensure_clean_store(self.path) as store:
 991            df = DataFrame(dict(A='foo', B='bar'), index=range(5))    991            df = DataFrame(dict(A='foo', B='bar'), index=range(5))
 992            df.loc[2, 'A'] = np.nan                                   992            df.loc[2, 'A'] = np.nan
 993            df.loc[3, 'B'] = np.nan                                   993            df.loc[3, 'B'] = np.nan
 994            _maybe_remove(store, 'df')                                994            _maybe_remove(store, 'df')
 995            store.append('df', df, encoding='ascii')                  995            store.append('df', df, encoding='ascii')
 996            tm.assert_frame_equal(store['df'], df)                    996            tm.assert_frame_equal(store['df'], df)
 997                                                                      997
 998            expected = df.reindex(columns=['A'])                      998            expected = df.reindex(columns=['A'])
 999            result = store.select('df', Term('columns=A', encoding=' 999            result = store.select('df', Term('columns=A', encoding='
                                                                ascii'))                                                                ascii'))
1000            tm.assert_frame_equal(result, expected)                  1000            tm.assert_frame_equal(result, expected)
1001                                                                     1001
1002    def test_latin_encoding(self):                                   1002    def test_latin_encoding(self):
1003                                                                     1003
1004        if compat.PY2:                                               1004        if compat.PY2:
1005            tm.assert_raises_regex(                                  1005            tm.assert_raises_regex(
1006                TypeError, r'\[unicode\] is not implemented as a tab1006                TypeError, r'\[unicode\] is not implemented as a tab
                                                             le column')                                                             le column')
1007            return                                                   1007            return
1008                                                                     1008
1009        values = [[b'E\xc9, 17', b'', b'a', b'b', b'c'],             1009        values = [[b'E\xc9, 17', b'', b'a', b'b', b'c'],
1010                  [b'E\xc9, 17', b'a', b'b', b'c'],                  1010                  [b'E\xc9, 17', b'a', b'b', b'c'],
1011                  [b'EE, 17', b'', b'a', b'b', b'c'],                1011                  [b'EE, 17', b'', b'a', b'b', b'c'],
1012                  [b'E\xc9, 17', b'\xf8\xfc', b'a', b'b', b'c'],     1012                  [b'E\xc9, 17', b'\xf8\xfc', b'a', b'b', b'c'],
1013                  [b'', b'a', b'b', b'c'],                           1013                  [b'', b'a', b'b', b'c'],
1014                  [b'\xf8\xfc', b'a', b'b', b'c'],                   1014                  [b'\xf8\xfc', b'a', b'b', b'c'],
1015                  [b'A\xf8\xfc', b'', b'a', b'b', b'c'],             1015                  [b'A\xf8\xfc', b'', b'a', b'b', b'c'],
1016                  [np.nan, b'', b'b', b'c'],                         1016                  [np.nan, b'', b'b', b'c'],
1017                  [b'A\xf8\xfc', np.nan, b'', b'b', b'c']]           1017                  [b'A\xf8\xfc', np.nan, b'', b'b', b'c']]
1018                                                                     1018
1019        def _try_decode(x, encoding='latin-1'):                      1019        def _try_decode(x, encoding='latin-1'):
1020            try:                                                     1020            try:
1021                return x.decode(encoding)                            1021                return x.decode(encoding)
1022            except AttributeError:                                   1022            except AttributeError:
1023                return x                                             1023                return x
1024        # not sure how to remove latin-1 from code in python 2 and 3 1024        # not sure how to remove latin-1 from code in python 2 and 3
1025        values = [[_try_decode(x) for x in y] for y in values]       1025        values = [[_try_decode(x) for x in y] for y in values]
1026                                                                     1026
1027        examples = []                                                1027        examples = []
1028        for dtype in ['category', object]:                           1028        for dtype in ['category', object]:
1029            for val in values:                                       1029            for val in values:
1030                examples.append(pandas.Series(val, dtype=dtype))     1030                examples.append(pandas.Series(val, dtype=dtype))
1031                                                                     1031
1032        def roundtrip(s, key='data', encoding='latin-1', nan_rep=''):1032        def roundtrip(s, key='data', encoding='latin-1', nan_rep=''):
1033            with ensure_clean_path(self.path) as store:              1033            with ensure_clean_path(self.path) as store:
1034                s.to_hdf(store, key, format='table', encoding=encodi1034                s.to_hdf(store, key, format='table', encoding=encodi
                                                                     ng,                                                                     ng,
1035                         nan_rep=nan_rep)                            1035                         nan_rep=nan_rep)
1036                retr = read_hdf(store, key)                          1036                retr = read_hdf(store, key)
1037                s_nan = s.replace(nan_rep, np.nan)                   1037                s_nan = s.replace(nan_rep, np.nan)
1038                assert_series_equal(s_nan, retr, check_categorical=F1038                assert_series_equal(s_nan, retr, check_categorical=F
                                                                   alse)                                                                   alse)
1039                                                                     1039
1040        for s in examples:                                           1040        for s in examples:
1041            roundtrip(s)                                             1041            roundtrip(s)
1042                                                                     1042
1043        # fails:                                                     1043        # fails:
1044        # for x in examples:                                         1044        # for x in examples:
1045        #     roundtrip(s, nan_rep=b'\xf8\xfc')                      1045        #     roundtrip(s, nan_rep=b'\xf8\xfc')
1046                                                                     1046
1047    def test_append_some_nans(self):                                 1047    def test_append_some_nans(self):
1048                                                                     1048
1049        with ensure_clean_store(self.path) as store:                 1049        with ensure_clean_store(self.path) as store:
1050            df = DataFrame({'A': Series(np.random.randn(20)).astype(1050            df = DataFrame({'A': Series(np.random.randn(20)).astype(
                                                               'int32'),                                                               'int32'),
1051                            'A1': np.random.randn(20),               1051                            'A1': np.random.randn(20),
1052                            'A2': np.random.randn(20),               1052                            'A2': np.random.randn(20),
1053                            'B': 'foo', 'C': 'bar',                  1053                            'B': 'foo', 'C': 'bar',
1054                            'D': Timestamp("20010101"),              1054                            'D': Timestamp("20010101"),
1055                            'E': datetime.datetime(2001, 1, 2, 0, 0)1055                            'E': datetime.datetime(2001, 1, 2, 0, 0)
                                                                      },                                                                      },
1056                           index=np.arange(20))                      1056                           index=np.arange(20))
1057            # some nans                                              1057            # some nans
1058            _maybe_remove(store, 'df1')                              1058            _maybe_remove(store, 'df1')
1059            df.loc[0:15, ['A1', 'B', 'D', 'E']] = np.nan             1059            df.loc[0:15, ['A1', 'B', 'D', 'E']] = np.nan
1060            store.append('df1', df[:10])                             1060            store.append('df1', df[:10])
1061            store.append('df1', df[10:])                             1061            store.append('df1', df[10:])
1062            tm.assert_frame_equal(store['df1'], df)                  1062            tm.assert_frame_equal(store['df1'], df)
1063                                                                     1063
1064            # first column                                           1064            # first column
1065            df1 = df.copy()                                          1065            df1 = df.copy()
1066            df1.loc[:, 'A1'] = np.nan                                1066            df1.loc[:, 'A1'] = np.nan
1067            _maybe_remove(store, 'df1')                              1067            _maybe_remove(store, 'df1')
1068            store.append('df1', df1[:10])                            1068            store.append('df1', df1[:10])
1069            store.append('df1', df1[10:])                            1069            store.append('df1', df1[10:])
1070            tm.assert_frame_equal(store['df1'], df1)                 1070            tm.assert_frame_equal(store['df1'], df1)
1071                                                                     1071
1072            # 2nd column                                             1072            # 2nd column
1073            df2 = df.copy()                                          1073            df2 = df.copy()
1074            df2.loc[:, 'A2'] = np.nan                                1074            df2.loc[:, 'A2'] = np.nan
1075            _maybe_remove(store, 'df2')                              1075            _maybe_remove(store, 'df2')
1076            store.append('df2', df2[:10])                            1076            store.append('df2', df2[:10])
1077            store.append('df2', df2[10:])                            1077            store.append('df2', df2[10:])
1078            tm.assert_frame_equal(store['df2'], df2)                 1078            tm.assert_frame_equal(store['df2'], df2)
1079                                                                     1079
1080            # datetimes                                              1080            # datetimes
1081            df3 = df.copy()                                          1081            df3 = df.copy()
1082            df3.loc[:, 'E'] = np.nan                                 1082            df3.loc[:, 'E'] = np.nan
1083            _maybe_remove(store, 'df3')                              1083            _maybe_remove(store, 'df3')
1084            store.append('df3', df3[:10])                            1084            store.append('df3', df3[:10])
1085            store.append('df3', df3[10:])                            1085            store.append('df3', df3[10:])
1086            tm.assert_frame_equal(store['df3'], df3)                 1086            tm.assert_frame_equal(store['df3'], df3)
1087                                                                     1087
1088    def test_append_all_nans(self):                                  1088    def test_append_all_nans(self):
1089                                                                     1089
1090        with ensure_clean_store(self.path) as store:                 1090        with ensure_clean_store(self.path) as store:
1091                                                                     1091
1092            df = DataFrame({'A1': np.random.randn(20),               1092            df = DataFrame({'A1': np.random.randn(20),
1093                            'A2': np.random.randn(20)},              1093                            'A2': np.random.randn(20)},
1094                           index=np.arange(20))                      1094                           index=np.arange(20))
1095            df.loc[0:15, :] = np.nan                                 1095            df.loc[0:15, :] = np.nan
1096                                                                     1096
1097            # nan some entire rows (dropna=True)                     1097            # nan some entire rows (dropna=True)
1098            _maybe_remove(store, 'df')                               1098            _maybe_remove(store, 'df')
1099            store.append('df', df[:10], dropna=True)                 1099            store.append('df', df[:10], dropna=True)
1100            store.append('df', df[10:], dropna=True)                 1100            store.append('df', df[10:], dropna=True)
1101            tm.assert_frame_equal(store['df'], df[-4:])              1101            tm.assert_frame_equal(store['df'], df[-4:])
1102                                                                     1102
1103            # nan some entire rows (dropna=False)                    1103            # nan some entire rows (dropna=False)
1104            _maybe_remove(store, 'df2')                              1104            _maybe_remove(store, 'df2')
1105            store.append('df2', df[:10], dropna=False)               1105            store.append('df2', df[:10], dropna=False)
1106            store.append('df2', df[10:], dropna=False)               1106            store.append('df2', df[10:], dropna=False)
1107            tm.assert_frame_equal(store['df2'], df)                  1107            tm.assert_frame_equal(store['df2'], df)
1108                                                                     1108
1109            # tests the option io.hdf.dropna_table                   1109            # tests the option io.hdf.dropna_table
1110            pandas.set_option('io.hdf.dropna_table', False)          1110            pandas.set_option('io.hdf.dropna_table', False)
1111            _maybe_remove(store, 'df3')                              1111            _maybe_remove(store, 'df3')
1112            store.append('df3', df[:10])                             1112            store.append('df3', df[:10])
1113            store.append('df3', df[10:])                             1113            store.append('df3', df[10:])
1114            tm.assert_frame_equal(store['df3'], df)                  1114            tm.assert_frame_equal(store['df3'], df)
1115                                                                     1115
1116            pandas.set_option('io.hdf.dropna_table', True)           1116            pandas.set_option('io.hdf.dropna_table', True)
1117            _maybe_remove(store, 'df4')                              1117            _maybe_remove(store, 'df4')
1118            store.append('df4', df[:10])                             1118            store.append('df4', df[:10])
1119            store.append('df4', df[10:])                             1119            store.append('df4', df[10:])
1120            tm.assert_frame_equal(store['df4'], df[-4:])             1120            tm.assert_frame_equal(store['df4'], df[-4:])
1121                                                                     1121
1122            # nan some entire rows (string are still written!)       1122            # nan some entire rows (string are still written!)
1123            df = DataFrame({'A1': np.random.randn(20),               1123            df = DataFrame({'A1': np.random.randn(20),
1124                            'A2': np.random.randn(20),               1124                            'A2': np.random.randn(20),
1125                            'B': 'foo', 'C': 'bar'},                 1125                            'B': 'foo', 'C': 'bar'},
1126                           index=np.arange(20))                      1126                           index=np.arange(20))
1127                                                                     1127
1128            df.loc[0:15, :] = np.nan                                 1128            df.loc[0:15, :] = np.nan
1129                                                                     1129
1130            _maybe_remove(store, 'df')                               1130            _maybe_remove(store, 'df')
1131            store.append('df', df[:10], dropna=True)                 1131            store.append('df', df[:10], dropna=True)
1132            store.append('df', df[10:], dropna=True)                 1132            store.append('df', df[10:], dropna=True)
1133            tm.assert_frame_equal(store['df'], df)                   1133            tm.assert_frame_equal(store['df'], df)
1134                                                                     1134
1135            _maybe_remove(store, 'df2')                              1135            _maybe_remove(store, 'df2')
1136            store.append('df2', df[:10], dropna=False)               1136            store.append('df2', df[:10], dropna=False)
1137            store.append('df2', df[10:], dropna=False)               1137            store.append('df2', df[10:], dropna=False)
1138            tm.assert_frame_equal(store['df2'], df)                  1138            tm.assert_frame_equal(store['df2'], df)
1139                                                                     1139
1140            # nan some entire rows (but since we have dates they are1140            # nan some entire rows (but since we have dates they are
                                                                   still                                                                   still
1141            # written!)                                              1141            # written!)
1142            df = DataFrame({'A1': np.random.randn(20),               1142            df = DataFrame({'A1': np.random.randn(20),
1143                            'A2': np.random.randn(20),               1143                            'A2': np.random.randn(20),
1144                            'B': 'foo', 'C': 'bar',                  1144                            'B': 'foo', 'C': 'bar',
1145                            'D': Timestamp("20010101"),              1145                            'D': Timestamp("20010101"),
1146                            'E': datetime.datetime(2001, 1, 2, 0, 0)1146                            'E': datetime.datetime(2001, 1, 2, 0, 0)
                                                                      },                                                                      },
1147                           index=np.arange(20))                      1147                           index=np.arange(20))
1148                                                                     1148
1149            df.loc[0:15, :] = np.nan                                 1149            df.loc[0:15, :] = np.nan
1150                                                                     1150
1151            _maybe_remove(store, 'df')                               1151            _maybe_remove(store, 'df')
1152            store.append('df', df[:10], dropna=True)                 1152            store.append('df', df[:10], dropna=True)
1153            store.append('df', df[10:], dropna=True)                 1153            store.append('df', df[10:], dropna=True)
1154            tm.assert_frame_equal(store['df'], df)                   1154            tm.assert_frame_equal(store['df'], df)
1155                                                                     1155
1156            _maybe_remove(store, 'df2')                              1156            _maybe_remove(store, 'df2')
1157            store.append('df2', df[:10], dropna=False)               1157            store.append('df2', df[:10], dropna=False)
1158            store.append('df2', df[10:], dropna=False)               1158            store.append('df2', df[10:], dropna=False)
1159            tm.assert_frame_equal(store['df2'], df)                  1159            tm.assert_frame_equal(store['df2'], df)
1160                                                                     1160
1161        # Test to make sure defaults are to not drop.                1161        # Test to make sure defaults are to not drop.
1162        # Corresponding to Issue 9382                                1162        # Corresponding to Issue 9382
1163        df_with_missing = DataFrame(                                 1163        df_with_missing = DataFrame(
1164            {'col1': [0, np.nan, 2], 'col2': [1, np.nan, np.nan]})   1164            {'col1': [0, np.nan, 2], 'col2': [1, np.nan, np.nan]})
1165                                                                     1165
1166        with ensure_clean_path(self.path) as path:                   1166        with ensure_clean_path(self.path) as path:
1167            df_with_missing.to_hdf(path, 'df_with_missing', format='1167            df_with_missing.to_hdf(path, 'df_with_missing', format='
                                                                 table')                                                                 table')
1168            reloaded = read_hdf(path, 'df_with_missing')             1168            reloaded = read_hdf(path, 'df_with_missing')
1169            tm.assert_frame_equal(df_with_missing, reloaded)         1169            tm.assert_frame_equal(df_with_missing, reloaded)
1170                                                                     1170
1171        matrix = [[[np.nan, np.nan, np.nan], [1, np.nan, np.nan]],   1171        matrix = [[[np.nan, np.nan, np.nan], [1, np.nan, np.nan]],
1172                  [[np.nan, np.nan, np.nan], [np.nan, 5, 6]],        1172                  [[np.nan, np.nan, np.nan], [np.nan, 5, 6]],
1173                  [[np.nan, np.nan, np.nan], [np.nan, 3, np.nan]]]   1173                  [[np.nan, np.nan, np.nan], [np.nan, 3, np.nan]]]
1174                                                                     1174
1175        with catch_warnings(record=True):                            1175        with catch_warnings(record=True):
1176            panel_with_missing = Panel(matrix,                       1176            panel_with_missing = Panel(matrix,
1177                                       items=['Item1', 'Item2', 'Ite1177                                       items=['Item1', 'Item2', 'Ite
                                                                   m3'],                                                                   m3'],
1178                                       major_axis=[1, 2],            1178                                       major_axis=[1, 2],
1179                                       minor_axis=['A', 'B', 'C'])   1179                                       minor_axis=['A', 'B', 'C'])
1180                                                                     1180
1181            with ensure_clean_path(self.path) as path:               1181            with ensure_clean_path(self.path) as path:
1182                panel_with_missing.to_hdf(                           1182                panel_with_missing.to_hdf(
1183                    path, 'panel_with_missing', format='table')      1183                    path, 'panel_with_missing', format='table')
1184                reloaded_panel = read_hdf(path, 'panel_with_missing')1184                reloaded_panel = read_hdf(path, 'panel_with_missing')
1185                tm.assert_panel_equal(panel_with_missing, reloaded_p1185                tm.assert_panel_equal(panel_with_missing, reloaded_p
                                                                   anel)                                                                   anel)
1186                                                                     1186
1187    def test_append_frame_column_oriented(self):                     1187    def test_append_frame_column_oriented(self):
1188                                                                     1188
1189        with ensure_clean_store(self.path) as store:                 1189        with ensure_clean_store(self.path) as store:
1190                                                                     1190
1191            # column oriented                                        1191            # column oriented
1192            df = tm.makeTimeDataFrame()                              1192            df = tm.makeTimeDataFrame()
1193            _maybe_remove(store, 'df1')                              1193            _maybe_remove(store, 'df1')
1194            store.append('df1', df.iloc[:, :2], axes=['columns'])    1194            store.append('df1', df.iloc[:, :2], axes=['columns'])
1195            store.append('df1', df.iloc[:, 2:])                      1195            store.append('df1', df.iloc[:, 2:])
1196            tm.assert_frame_equal(store['df1'], df)                  1196            tm.assert_frame_equal(store['df1'], df)
1197                                                                     1197
1198            result = store.select('df1', 'columns=A')                1198            result = store.select('df1', 'columns=A')
1199            expected = df.reindex(columns=['A'])                     1199            expected = df.reindex(columns=['A'])
1200            tm.assert_frame_equal(expected, result)                  1200            tm.assert_frame_equal(expected, result)
1201                                                                     1201
1202            # selection on the non-indexable                         1202            # selection on the non-indexable
1203            result = store.select(                                   1203            result = store.select(
1204                'df1', ('columns=A', 'index=df.index[0:4]'))         1204                'df1', ('columns=A', 'index=df.index[0:4]'))
1205            expected = df.reindex(columns=['A'], index=df.index[0:4])1205            expected = df.reindex(columns=['A'], index=df.index[0:4])
1206            tm.assert_frame_equal(expected, result)                  1206            tm.assert_frame_equal(expected, result)
1207                                                                     1207
1208            # this isn't supported                                   1208            # this isn't supported
1209            with pytest.raises(TypeError):                           1209            with pytest.raises(TypeError):
1210                store.select('df1',                                  1210                store.select('df1',
1211                             'columns=A and index>df.index[4]')      1211                             'columns=A and index>df.index[4]')
1212                                                                     1212
1213    def test_append_with_different_block_ordering(self):             1213    def test_append_with_different_block_ordering(self):
1214                                                                     1214
1215        # GH 4096; using same frames, but different block orderings  1215        # GH 4096; using same frames, but different block orderings
1216        with ensure_clean_store(self.path) as store:                 1216        with ensure_clean_store(self.path) as store:
1217                                                                     1217
1218            for i in range(10):                                      1218            for i in range(10):
1219                                                                     1219
1220                df = DataFrame(np.random.randn(10, 2), columns=list(1220                df = DataFrame(np.random.randn(10, 2), columns=list(
                                                                  'AB'))                                                                  'AB'))
1221                df['index'] = range(10)                              1221                df['index'] = range(10)
1222                df['index'] += i * 10                                1222                df['index'] += i * 10
1223                df['int64'] = Series([1] * len(df), dtype='int64')   1223                df['int64'] = Series([1] * len(df), dtype='int64')
1224                df['int16'] = Series([1] * len(df), dtype='int16')   1224                df['int16'] = Series([1] * len(df), dtype='int16')
1225                                                                     1225
1226                if i % 2 == 0:                                       1226                if i % 2 == 0:
1227                    del df['int64']                                  1227                    del df['int64']
1228                    df['int64'] = Series([1] * len(df), dtype='int641228                    df['int64'] = Series([1] * len(df), dtype='int64
                                                                      ')                                                                      ')
1229                if i % 3 == 0:                                       1229                if i % 3 == 0:
1230                    a = df.pop('A')                                  1230                    a = df.pop('A')
1231                    df['A'] = a                                      1231                    df['A'] = a
1232                                                                     1232
1233                df.set_index('index', inplace=True)                  1233                df.set_index('index', inplace=True)
1234                                                                     1234
1235                store.append('df', df)                               1235                store.append('df', df)
1236                                                                     1236
1237        # test a different ordering but with more fields (like inval1237        # test a different ordering but with more fields (like inval
                                                                      id                                                                      id
1238        # combinate)                                                 1238        # combinate)
1239        with ensure_clean_store(self.path) as store:                 1239        with ensure_clean_store(self.path) as store:
1240                                                                     1240
1241            df = DataFrame(np.random.randn(10, 2),                   1241            df = DataFrame(np.random.randn(10, 2),
1242                           columns=list('AB'), dtype='float64')      1242                           columns=list('AB'), dtype='float64')
1243            df['int64'] = Series([1] * len(df), dtype='int64')       1243            df['int64'] = Series([1] * len(df), dtype='int64')
1244            df['int16'] = Series([1] * len(df), dtype='int16')       1244            df['int16'] = Series([1] * len(df), dtype='int16')
1245            store.append('df', df)                                   1245            store.append('df', df)
1246                                                                     1246
1247            # store additonal fields in different blocks             1247            # store additonal fields in different blocks
1248            df['int16_2'] = Series([1] * len(df), dtype='int16')     1248            df['int16_2'] = Series([1] * len(df), dtype='int16')
1249            pytest.raises(ValueError, store.append, 'df', df)        1249            pytest.raises(ValueError, store.append, 'df', df)
1250                                                                     1250
1251            # store multile additonal fields in different blocks     1251            # store multile additonal fields in different blocks
1252            df['float_3'] = Series([1.] * len(df), dtype='float64')  1252            df['float_3'] = Series([1.] * len(df), dtype='float64')
1253            pytest.raises(ValueError, store.append, 'df', df)        1253            pytest.raises(ValueError, store.append, 'df', df)
1254                                                                     1254
1255    def test_ndim_indexables(self):                                  1255    def test_ndim_indexables(self):
1256        # test using ndim tables in new ways                         1256        # test using ndim tables in new ways
1257                                                                     1257
1258        with catch_warnings(record=True):                            1258        with catch_warnings(record=True):
1259            with ensure_clean_store(self.path) as store:             1259            with ensure_clean_store(self.path) as store:
1260                                                                     1260
1261                p4d = tm.makePanel4D()                               1261                p4d = tm.makePanel4D()
1262                                                                     1262
1263                def check_indexers(key, indexers):                   1263                def check_indexers(key, indexers):
1264                    for i, idx in enumerate(indexers):               1264                    for i, idx in enumerate(indexers):
1265                        descr = getattr(store.root, key).table.descr1265                        descr = getattr(store.root, key).table.descr
                                                                  iption                                                                  iption
1266                        assert getattr(descr, idx)._v_pos == i       1266                        assert getattr(descr, idx)._v_pos == i
1267                                                                     1267
1268                # append then change (will take existing schema)     1268                # append then change (will take existing schema)
1269                indexers = ['items', 'major_axis', 'minor_axis']     1269                indexers = ['items', 'major_axis', 'minor_axis']
1270                                                                     1270
1271                _maybe_remove(store, 'p4d')                          1271                _maybe_remove(store, 'p4d')
1272                store.append('p4d', p4d.iloc[:, :, :10, :], axes=ind1272                store.append('p4d', p4d.iloc[:, :, :10, :], axes=ind
                                                                  exers)                                                                  exers)
1273                store.append('p4d', p4d.iloc[:, :, 10:, :])          1273                store.append('p4d', p4d.iloc[:, :, 10:, :])
1274                assert_panel4d_equal(store.select('p4d'), p4d)       1274                assert_panel4d_equal(store.select('p4d'), p4d)
1275                check_indexers('p4d', indexers)                      1275                check_indexers('p4d', indexers)
1276                                                                     1276
1277                # same as above, but try to append with differnt axes1277                # same as above, but try to append with differnt axes
1278                _maybe_remove(store, 'p4d')                          1278                _maybe_remove(store, 'p4d')
1279                store.append('p4d', p4d.iloc[:, :, :10, :], axes=ind1279                store.append('p4d', p4d.iloc[:, :, :10, :], axes=ind
                                                                  exers)                                                                  exers)
1280                store.append('p4d', p4d.iloc[:, :, 10:, :], axes=[   1280                store.append('p4d', p4d.iloc[:, :, 10:, :], axes=[
1281                    'labels', 'items', 'major_axis'])                1281                    'labels', 'items', 'major_axis'])
1282                assert_panel4d_equal(store.select('p4d'), p4d)       1282                assert_panel4d_equal(store.select('p4d'), p4d)
1283                check_indexers('p4d', indexers)                      1283                check_indexers('p4d', indexers)
1284                                                                     1284
1285                # pass incorrect number of axes                      1285                # pass incorrect number of axes
1286                _maybe_remove(store, 'p4d')                          1286                _maybe_remove(store, 'p4d')
1287                pytest.raises(ValueError, store.append, 'p4d', p4d.i1287                pytest.raises(ValueError, store.append, 'p4d', p4d.i
                                                                    loc[                                                                    loc[
1288                    :, :, :10, :], axes=['major_axis', 'minor_axis'])1288                    :, :, :10, :], axes=['major_axis', 'minor_axis'])
1289                                                                     1289
1290                # different than default indexables #1               1290                # different than default indexables #1
1291                indexers = ['labels', 'major_axis', 'minor_axis']    1291                indexers = ['labels', 'major_axis', 'minor_axis']
1292                _maybe_remove(store, 'p4d')                          1292                _maybe_remove(store, 'p4d')
1293                store.append('p4d', p4d.iloc[:, :, :10, :], axes=ind1293                store.append('p4d', p4d.iloc[:, :, :10, :], axes=ind
                                                                  exers)                                                                  exers)
1294                store.append('p4d', p4d.iloc[:, :, 10:, :])          1294                store.append('p4d', p4d.iloc[:, :, 10:, :])
1295                assert_panel4d_equal(store['p4d'], p4d)              1295                assert_panel4d_equal(store['p4d'], p4d)
1296                check_indexers('p4d', indexers)                      1296                check_indexers('p4d', indexers)
1297                                                                     1297
1298                # different than default indexables #2               1298                # different than default indexables #2
1299                indexers = ['major_axis', 'labels', 'minor_axis']    1299                indexers = ['major_axis', 'labels', 'minor_axis']
1300                _maybe_remove(store, 'p4d')                          1300                _maybe_remove(store, 'p4d')
1301                store.append('p4d', p4d.iloc[:, :, :10, :], axes=ind1301                store.append('p4d', p4d.iloc[:, :, :10, :], axes=ind
                                                                  exers)                                                                  exers)
1302                store.append('p4d', p4d.iloc[:, :, 10:, :])          1302                store.append('p4d', p4d.iloc[:, :, 10:, :])
1303                assert_panel4d_equal(store['p4d'], p4d)              1303                assert_panel4d_equal(store['p4d'], p4d)
1304                check_indexers('p4d', indexers)                      1304                check_indexers('p4d', indexers)
1305                                                                     1305
1306                # partial selection                                  1306                # partial selection
1307                result = store.select('p4d', ['labels=l1'])          1307                result = store.select('p4d', ['labels=l1'])
1308                expected = p4d.reindex(labels=['l1'])                1308                expected = p4d.reindex(labels=['l1'])
1309                assert_panel4d_equal(result, expected)               1309                assert_panel4d_equal(result, expected)
1310                                                                     1310
1311                # partial selection2                                 1311                # partial selection2
1312                result = store.select(                               1312                result = store.select(
1313                    'p4d', "labels='l1' and items='ItemA' and minor_1313                    'p4d', "labels='l1' and items='ItemA' and minor_
                                                              axis='B'")                                                              axis='B'")
1314                expected = p4d.reindex(                              1314                expected = p4d.reindex(
1315                    labels=['l1'], items=['ItemA'], minor_axis=['B'])1315                    labels=['l1'], items=['ItemA'], minor_axis=['B'])
1316                assert_panel4d_equal(result, expected)               1316                assert_panel4d_equal(result, expected)
1317                                                                     1317
1318                # non-existant partial selection                     1318                # non-existant partial selection
1319                result = store.select(                               1319                result = store.select(
1320                    'p4d', "labels='l1' and items='Item1' and minor_1320                    'p4d', "labels='l1' and items='Item1' and minor_
                                                              axis='B'")                                                              axis='B'")
1321                expected = p4d.reindex(labels=['l1'], items=[],      1321                expected = p4d.reindex(labels=['l1'], items=[],
1322                                       minor_axis=['B'])             1322                                       minor_axis=['B'])
1323                assert_panel4d_equal(result, expected)               1323                assert_panel4d_equal(result, expected)
1324                                                                     1324
1325    def test_append_with_strings(self):                              1325    def test_append_with_strings(self):
1326                                                                     1326
1327        with ensure_clean_store(self.path) as store:                 1327        with ensure_clean_store(self.path) as store:
1328            with catch_warnings(record=True):                        1328            with catch_warnings(record=True):
1329                wp = tm.makePanel()                                  1329                wp = tm.makePanel()
1330                wp2 = wp.rename_axis(                                1330                wp2 = wp.rename_axis(
1331                    dict([(x, "%s_extra" % x) for x in wp.minor_axis1331                    dict([(x, "%s_extra" % x) for x in wp.minor_axis
                                                             ]), axis=2)                                                             ]), axis=2)
1332                                                                     1332
1333                def check_col(key, name, size):                      1333                def check_col(key, name, size):
1334                    assert getattr(store.get_storer(key)             1334                    assert getattr(store.get_storer(key)
1335                                   .table.description, name).itemsiz1335                                   .table.description, name).itemsiz
                                                               e == size                                                               e == size
1336                                                                     1336
1337                store.append('s1', wp, min_itemsize=20)              1337                store.append('s1', wp, min_itemsize=20)
1338                store.append('s1', wp2)                              1338                store.append('s1', wp2)
1339                expected = concat([wp, wp2], axis=2)                 1339                expected = concat([wp, wp2], axis=2)
1340                expected = expected.reindex(                         1340                expected = expected.reindex(
1341                    minor_axis=sorted(expected.minor_axis))          1341                    minor_axis=sorted(expected.minor_axis))
1342                assert_panel_equal(store['s1'], expected)            1342                assert_panel_equal(store['s1'], expected)
1343                check_col('s1', 'minor_axis', 20)                    1343                check_col('s1', 'minor_axis', 20)
1344                                                                     1344
1345                # test dict format                                   1345                # test dict format
1346                store.append('s2', wp, min_itemsize={'minor_axis': 21346                store.append('s2', wp, min_itemsize={'minor_axis': 2
                                                                     0})                                                                     0})
1347                store.append('s2', wp2)                              1347                store.append('s2', wp2)
1348                expected = concat([wp, wp2], axis=2)                 1348                expected = concat([wp, wp2], axis=2)
1349                expected = expected.reindex(                         1349                expected = expected.reindex(
1350                    minor_axis=sorted(expected.minor_axis))          1350                    minor_axis=sorted(expected.minor_axis))
1351                assert_panel_equal(store['s2'], expected)            1351                assert_panel_equal(store['s2'], expected)
1352                check_col('s2', 'minor_axis', 20)                    1352                check_col('s2', 'minor_axis', 20)
1353                                                                     1353
1354                # apply the wrong field (similar to #1)              1354                # apply the wrong field (similar to #1)
1355                store.append('s3', wp, min_itemsize={'major_axis': 21355                store.append('s3', wp, min_itemsize={'major_axis': 2
                                                                     0})                                                                     0})
1356                pytest.raises(ValueError, store.append, 's3', wp2)   1356                pytest.raises(ValueError, store.append, 's3', wp2)
1357                                                                     1357
1358                # test truncation of bigger strings                  1358                # test truncation of bigger strings
1359                store.append('s4', wp)                               1359                store.append('s4', wp)
1360                pytest.raises(ValueError, store.append, 's4', wp2)   1360                pytest.raises(ValueError, store.append, 's4', wp2)
1361                                                                     1361
1362                # avoid truncation on elements                       1362                # avoid truncation on elements
1363                df = DataFrame([[123, 'asdqwerty'], [345, 'dggnhebbs1363                df = DataFrame([[123, 'asdqwerty'], [345, 'dggnhebbs
                                                              dfbdfb']])                                                              dfbdfb']])
1364                store.append('df_big', df)                           1364                store.append('df_big', df)
1365                tm.assert_frame_equal(store.select('df_big'), df)    1365                tm.assert_frame_equal(store.select('df_big'), df)
1366                check_col('df_big', 'values_block_1', 15)            1366                check_col('df_big', 'values_block_1', 15)
1367                                                                     1367
1368                # appending smaller string ok                        1368                # appending smaller string ok
1369                df2 = DataFrame([[124, 'asdqy'], [346, 'dggnhefbdfb'1369                df2 = DataFrame([[124, 'asdqy'], [346, 'dggnhefbdfb'
                                                                     ]])                                                                     ]])
1370                store.append('df_big', df2)                          1370                store.append('df_big', df2)
1371                expected = concat([df, df2])                         1371                expected = concat([df, df2])
1372                tm.assert_frame_equal(store.select('df_big'), expect1372                tm.assert_frame_equal(store.select('df_big'), expect
                                                                     ed)                                                                     ed)
1373                check_col('df_big', 'values_block_1', 15)            1373                check_col('df_big', 'values_block_1', 15)
1374                                                                     1374
1375                # avoid truncation on elements                       1375                # avoid truncation on elements
1376                df = DataFrame([[123, 'asdqwerty'], [345, 'dggnhebbs1376                df = DataFrame([[123, 'asdqwerty'], [345, 'dggnhebbs
                                                              dfbdfb']])                                                              dfbdfb']])
1377                store.append('df_big2', df, min_itemsize={'values': 1377                store.append('df_big2', df, min_itemsize={'values': 
                                                                    50})                                                                    50})
1378                tm.assert_frame_equal(store.select('df_big2'), df)   1378                tm.assert_frame_equal(store.select('df_big2'), df)
1379                check_col('df_big2', 'values_block_1', 50)           1379                check_col('df_big2', 'values_block_1', 50)
1380                                                                     1380
1381                # bigger string on next append                       1381                # bigger string on next append
1382                store.append('df_new', df)                           1382                store.append('df_new', df)
1383                df_new = DataFrame(                                  1383                df_new = DataFrame(
1384                    [[124, 'abcdefqhij'], [346, 'abcdefghijklmnopqrt1384                    [[124, 'abcdefqhij'], [346, 'abcdefghijklmnopqrt
                                                             suvwxyz']])                                                             suvwxyz']])
1385                pytest.raises(ValueError, store.append, 'df_new', df1385                pytest.raises(ValueError, store.append, 'df_new', df
                                                                   _new)                                                                   _new)
1386                                                                     1386
1387                # min_itemsize on Series index (GH 11412)            1387                # min_itemsize on Series index (GH 11412)
1388                df = tm.makeMixedDataFrame().set_index('C')          1388                df = tm.makeMixedDataFrame().set_index('C')
1389                store.append('ss', df['B'], min_itemsize={'index': 41389                store.append('ss', df['B'], min_itemsize={'index': 4
                                                                      })                                                                      })
1390                tm.assert_series_equal(store.select('ss'), df['B'])  1390                tm.assert_series_equal(store.select('ss'), df['B'])
1391                                                                     1391
1392                # same as above, with data_columns=True              1392                # same as above, with data_columns=True
1393                store.append('ss2', df['B'], data_columns=True,      1393                store.append('ss2', df['B'], data_columns=True,
1394                             min_itemsize={'index': 4})              1394                             min_itemsize={'index': 4})
1395                tm.assert_series_equal(store.select('ss2'), df['B']) 1395                tm.assert_series_equal(store.select('ss2'), df['B'])
1396                                                                     1396
1397                # min_itemsize in index without appending (GH 10381) 1397                # min_itemsize in index without appending (GH 10381)
1398                store.put('ss3', df, format='table',                 1398                store.put('ss3', df, format='table',
1399                          min_itemsize={'index': 6})                 1399                          min_itemsize={'index': 6})
1400                # just make sure there is a longer string:           1400                # just make sure there is a longer string:
1401                df2 = df.copy().reset_index().assign(C='longer').set1401                df2 = df.copy().reset_index().assign(C='longer').set
                                                             _index('C')                                                             _index('C')
1402                store.append('ss3', df2)                             1402                store.append('ss3', df2)
1403                tm.assert_frame_equal(store.select('ss3'),           1403                tm.assert_frame_equal(store.select('ss3'),
1404                                      pd.concat([df, df2]))          1404                                      pd.concat([df, df2]))
1405                                                                     1405
1406                # same as above, with a Series                       1406                # same as above, with a Series
1407                store.put('ss4', df['B'], format='table',            1407                store.put('ss4', df['B'], format='table',
1408                          min_itemsize={'index': 6})                 1408                          min_itemsize={'index': 6})
1409                store.append('ss4', df2['B'])                        1409                store.append('ss4', df2['B'])
1410                tm.assert_series_equal(store.select('ss4'),          1410                tm.assert_series_equal(store.select('ss4'),
1411                                       pd.concat([df['B'], df2['B']]1411                                       pd.concat([df['B'], df2['B']]
                                                                      ))                                                                      ))
1412                                                                     1412
1413                # with nans                                          1413                # with nans
1414                _maybe_remove(store, 'df')                           1414                _maybe_remove(store, 'df')
1415                df = tm.makeTimeDataFrame()                          1415                df = tm.makeTimeDataFrame()
1416                df['string'] = 'foo'                                 1416                df['string'] = 'foo'
1417                df.loc[1:4, 'string'] = np.nan                       1417                df.loc[1:4, 'string'] = np.nan
1418                df['string2'] = 'bar'                                1418                df['string2'] = 'bar'
1419                df.loc[4:8, 'string2'] = np.nan                      1419                df.loc[4:8, 'string2'] = np.nan
1420                df['string3'] = 'bah'                                1420                df['string3'] = 'bah'
1421                df.loc[1:, 'string3'] = np.nan                       1421                df.loc[1:, 'string3'] = np.nan
1422                store.append('df', df)                               1422                store.append('df', df)
1423                result = store.select('df')                          1423                result = store.select('df')
1424                tm.assert_frame_equal(result, df)                    1424                tm.assert_frame_equal(result, df)
1425                                                                     1425
1426        with ensure_clean_store(self.path) as store:                 1426        with ensure_clean_store(self.path) as store:
1427                                                                     1427
1428            def check_col(key, name, size):                          1428            def check_col(key, name, size):
1429                assert getattr(store.get_storer(key)                 1429                assert getattr(store.get_storer(key)
1430                               .table.description, name).itemsize, s1430                               .table.description, name).itemsize, s
                                                                     ize                                                                     ize
1431                                                                     1431
1432            df = DataFrame(dict(A='foo', B='bar'), index=range(10))  1432            df = DataFrame(dict(A='foo', B='bar'), index=range(10))
1433                                                                     1433
1434            # a min_itemsize that creates a data_column              1434            # a min_itemsize that creates a data_column
1435            _maybe_remove(store, 'df')                               1435            _maybe_remove(store, 'df')
1436            store.append('df', df, min_itemsize={'A': 200})          1436            store.append('df', df, min_itemsize={'A': 200})
1437            check_col('df', 'A', 200)                                1437            check_col('df', 'A', 200)
1438            assert store.get_storer('df').data_columns == ['A']      1438            assert store.get_storer('df').data_columns == ['A']
1439                                                                     1439
1440            # a min_itemsize that creates a data_column2             1440            # a min_itemsize that creates a data_column2
1441            _maybe_remove(store, 'df')                               1441            _maybe_remove(store, 'df')
1442            store.append('df', df, data_columns=['B'], min_itemsize=1442            store.append('df', df, data_columns=['B'], min_itemsize=
                                                             {'A': 200})                                                             {'A': 200})
1443            check_col('df', 'A', 200)                                1443            check_col('df', 'A', 200)
1444            assert store.get_storer('df').data_columns == ['B', 'A'] 1444            assert store.get_storer('df').data_columns == ['B', 'A']
1445                                                                     1445
1446            # a min_itemsize that creates a data_column2             1446            # a min_itemsize that creates a data_column2
1447            _maybe_remove(store, 'df')                               1447            _maybe_remove(store, 'df')
1448            store.append('df', df, data_columns=[                    1448            store.append('df', df, data_columns=[
1449                         'B'], min_itemsize={'values': 200})         1449                         'B'], min_itemsize={'values': 200})
1450            check_col('df', 'B', 200)                                1450            check_col('df', 'B', 200)
1451            check_col('df', 'values_block_0', 200)                   1451            check_col('df', 'values_block_0', 200)
1452            assert store.get_storer('df').data_columns == ['B']      1452            assert store.get_storer('df').data_columns == ['B']
1453                                                                     1453
1454            # infer the .typ on subsequent appends                   1454            # infer the .typ on subsequent appends
1455            _maybe_remove(store, 'df')                               1455            _maybe_remove(store, 'df')
1456            store.append('df', df[:5], min_itemsize=200)             1456            store.append('df', df[:5], min_itemsize=200)
1457            store.append('df', df[5:], min_itemsize=200)             1457            store.append('df', df[5:], min_itemsize=200)
1458            tm.assert_frame_equal(store['df'], df)                   1458            tm.assert_frame_equal(store['df'], df)
1459                                                                     1459
1460            # invalid min_itemsize keys                              1460            # invalid min_itemsize keys
1461            df = DataFrame(['foo', 'foo', 'foo', 'barh',             1461            df = DataFrame(['foo', 'foo', 'foo', 'barh',
1462                            'barh', 'barh'], columns=['A'])          1462                            'barh', 'barh'], columns=['A'])
1463            _maybe_remove(store, 'df')                               1463            _maybe_remove(store, 'df')
1464            pytest.raises(ValueError, store.append, 'df',            1464            pytest.raises(ValueError, store.append, 'df',
1465                          df, min_itemsize={'foo': 20, 'foobar': 20})1465                          df, min_itemsize={'foo': 20, 'foobar': 20})
1466                                                                     1466
1467    def test_to_hdf_with_min_itemsize(self):                         1467    def test_to_hdf_with_min_itemsize(self):
1468                                                                     1468
1469        with ensure_clean_path(self.path) as path:                   1469        with ensure_clean_path(self.path) as path:
1470                                                                     1470
1471            # min_itemsize in index with to_hdf (GH 10381)           1471            # min_itemsize in index with to_hdf (GH 10381)
1472            df = tm.makeMixedDataFrame().set_index('C')              1472            df = tm.makeMixedDataFrame().set_index('C')
1473            df.to_hdf(path, 'ss3', format='table', min_itemsize={'in1473            df.to_hdf(path, 'ss3', format='table', min_itemsize={'in
                                                               dex': 6})                                                               dex': 6})
1474            # just make sure there is a longer string:               1474            # just make sure there is a longer string:
1475            df2 = df.copy().reset_index().assign(C='longer').set_ind1475            df2 = df.copy().reset_index().assign(C='longer').set_ind
                                                                 ex('C')                                                                 ex('C')
1476            df2.to_hdf(path, 'ss3', append=True, format='table')     1476            df2.to_hdf(path, 'ss3', append=True, format='table')
1477            tm.assert_frame_equal(pd.read_hdf(path, 'ss3'),          1477            tm.assert_frame_equal(pd.read_hdf(path, 'ss3'),
1478                                  pd.concat([df, df2]))              1478                                  pd.concat([df, df2]))
1479                                                                     1479
1480            # same as above, with a Series                           1480            # same as above, with a Series
1481            df['B'].to_hdf(path, 'ss4', format='table',              1481            df['B'].to_hdf(path, 'ss4', format='table',
1482                           min_itemsize={'index': 6})                1482                           min_itemsize={'index': 6})
1483            df2['B'].to_hdf(path, 'ss4', append=True, format='table')1483            df2['B'].to_hdf(path, 'ss4', append=True, format='table')
1484            tm.assert_series_equal(pd.read_hdf(path, 'ss4'),         1484            tm.assert_series_equal(pd.read_hdf(path, 'ss4'),
1485                                   pd.concat([df['B'], df2['B']]))   1485                                   pd.concat([df['B'], df2['B']]))
1486                                                                     1486
1487    def test_append_with_data_columns(self):                         1487    def test_append_with_data_columns(self):
1488                                                                     1488
1489        with ensure_clean_store(self.path) as store:                 1489        with ensure_clean_store(self.path) as store:
1490            df = tm.makeTimeDataFrame()                              1490            df = tm.makeTimeDataFrame()
1491            df.iloc[0, df.columns.get_loc('B')] = 1.                 1491            df.iloc[0, df.columns.get_loc('B')] = 1.
1492            _maybe_remove(store, 'df')                               1492            _maybe_remove(store, 'df')
1493            store.append('df', df[:2], data_columns=['B'])           1493            store.append('df', df[:2], data_columns=['B'])
1494            store.append('df', df[2:])                               1494            store.append('df', df[2:])
1495            tm.assert_frame_equal(store['df'], df)                   1495            tm.assert_frame_equal(store['df'], df)
1496                                                                     1496
1497            # check that we have indicies created                    1497            # check that we have indicies created
1498            assert(store._handle.root.df.table.cols.index.is_indexed1498            assert(store._handle.root.df.table.cols.index.is_indexed
                                                                is True)                                                                is True)
1499            assert(store._handle.root.df.table.cols.B.is_indexed is 1499            assert(store._handle.root.df.table.cols.B.is_indexed is 
                                                                   True)                                                                   True)
1500                                                                     1500
1501            # data column searching                                  1501            # data column searching
1502            result = store.select('df', 'B>0')                       1502            result = store.select('df', 'B>0')
1503            expected = df[df.B > 0]                                  1503            expected = df[df.B > 0]
1504            tm.assert_frame_equal(result, expected)                  1504            tm.assert_frame_equal(result, expected)
1505                                                                     1505
1506            # data column searching (with an indexable and a data_co1506            # data column searching (with an indexable and a data_co
                                                                  lumns)                                                                  lumns)
1507            result = store.select(                                   1507            result = store.select(
1508                'df', 'B>0 and index>df.index[3]')                   1508                'df', 'B>0 and index>df.index[3]')
1509            df_new = df.reindex(index=df.index[4:])                  1509            df_new = df.reindex(index=df.index[4:])
1510            expected = df_new[df_new.B > 0]                          1510            expected = df_new[df_new.B > 0]
1511            tm.assert_frame_equal(result, expected)                  1511            tm.assert_frame_equal(result, expected)
1512                                                                     1512
1513            # data column selection with a string data_column        1513            # data column selection with a string data_column
1514            df_new = df.copy()                                       1514            df_new = df.copy()
1515            df_new['string'] = 'foo'                                 1515            df_new['string'] = 'foo'
1516            df_new.loc[1:4, 'string'] = np.nan                       1516            df_new.loc[1:4, 'string'] = np.nan
1517            df_new.loc[5:6, 'string'] = 'bar'                        1517            df_new.loc[5:6, 'string'] = 'bar'
1518            _maybe_remove(store, 'df')                               1518            _maybe_remove(store, 'df')
1519            store.append('df', df_new, data_columns=['string'])      1519            store.append('df', df_new, data_columns=['string'])
1520            result = store.select('df', "string='foo'")              1520            result = store.select('df', "string='foo'")
1521            expected = df_new[df_new.string == 'foo']                1521            expected = df_new[df_new.string == 'foo']
1522            tm.assert_frame_equal(result, expected)                  1522            tm.assert_frame_equal(result, expected)
1523                                                                     1523
1524            # using min_itemsize and a data column                   1524            # using min_itemsize and a data column
1525            def check_col(key, name, size):                          1525            def check_col(key, name, size):
1526                assert getattr(store.get_storer(key)                 1526                assert getattr(store.get_storer(key)
1527                               .table.description, name).itemsize ==1527                               .table.description, name).itemsize ==
                                                                    size                                                                    size
1528                                                                     1528
1529        with ensure_clean_store(self.path) as store:                 1529        with ensure_clean_store(self.path) as store:
1530            _maybe_remove(store, 'df')                               1530            _maybe_remove(store, 'df')
1531            store.append('df', df_new, data_columns=['string'],      1531            store.append('df', df_new, data_columns=['string'],
1532                         min_itemsize={'string': 30})                1532                         min_itemsize={'string': 30})
1533            check_col('df', 'string', 30)                            1533            check_col('df', 'string', 30)
1534            _maybe_remove(store, 'df')                               1534            _maybe_remove(store, 'df')
1535            store.append(                                            1535            store.append(
1536                'df', df_new, data_columns=['string'], min_itemsize=1536                'df', df_new, data_columns=['string'], min_itemsize=
                                                                     30)                                                                     30)
1537            check_col('df', 'string', 30)                            1537            check_col('df', 'string', 30)
1538            _maybe_remove(store, 'df')                               1538            _maybe_remove(store, 'df')
1539            store.append('df', df_new, data_columns=['string'],      1539            store.append('df', df_new, data_columns=['string'],
1540                         min_itemsize={'values': 30})                1540                         min_itemsize={'values': 30})
1541            check_col('df', 'string', 30)                            1541            check_col('df', 'string', 30)
1542                                                                     1542
1543        with ensure_clean_store(self.path) as store:                 1543        with ensure_clean_store(self.path) as store:
1544            df_new['string2'] = 'foobarbah'                          1544            df_new['string2'] = 'foobarbah'
1545            df_new['string_block1'] = 'foobarbah1'                   1545            df_new['string_block1'] = 'foobarbah1'
1546            df_new['string_block2'] = 'foobarbah2'                   1546            df_new['string_block2'] = 'foobarbah2'
1547            _maybe_remove(store, 'df')                               1547            _maybe_remove(store, 'df')
1548            store.append('df', df_new, data_columns=['string', 'stri1548            store.append('df', df_new, data_columns=['string', 'stri
                                                                  ng2'],                                                                  ng2'],
1549                         min_itemsize={'string': 30, 'string2': 40,  1549                         min_itemsize={'string': 30, 'string2': 40,
1550                                       'values': 50})                1550                                       'values': 50})
1551            check_col('df', 'string', 30)                            1551            check_col('df', 'string', 30)
1552            check_col('df', 'string2', 40)                           1552            check_col('df', 'string2', 40)
1553            check_col('df', 'values_block_1', 50)                    1553            check_col('df', 'values_block_1', 50)
1554                                                                     1554
1555        with ensure_clean_store(self.path) as store:                 1555        with ensure_clean_store(self.path) as store:
1556            # multiple data columns                                  1556            # multiple data columns
1557            df_new = df.copy()                                       1557            df_new = df.copy()
1558            df_new.iloc[0, df_new.columns.get_loc('A')] = 1.         1558            df_new.iloc[0, df_new.columns.get_loc('A')] = 1.
1559            df_new.iloc[0, df_new.columns.get_loc('B')] = -1.        1559            df_new.iloc[0, df_new.columns.get_loc('B')] = -1.
1560            df_new['string'] = 'foo'                                 1560            df_new['string'] = 'foo'
1561                                                                     1561
1562            sl = df_new.columns.get_loc('string')                    1562            sl = df_new.columns.get_loc('string')
1563            df_new.iloc[1:4, sl] = np.nan                            1563            df_new.iloc[1:4, sl] = np.nan
1564            df_new.iloc[5:6, sl] = 'bar'                             1564            df_new.iloc[5:6, sl] = 'bar'
1565                                                                     1565
1566            df_new['string2'] = 'foo'                                1566            df_new['string2'] = 'foo'
1567            sl = df_new.columns.get_loc('string2')                   1567            sl = df_new.columns.get_loc('string2')
1568            df_new.iloc[2:5, sl] = np.nan                            1568            df_new.iloc[2:5, sl] = np.nan
1569            df_new.iloc[7:8, sl] = 'bar'                             1569            df_new.iloc[7:8, sl] = 'bar'
1570            _maybe_remove(store, 'df')                               1570            _maybe_remove(store, 'df')
1571            store.append(                                            1571            store.append(
1572                'df', df_new, data_columns=['A', 'B', 'string', 'str1572                'df', df_new, data_columns=['A', 'B', 'string', 'str
                                                                 ing2'])                                                                 ing2'])
1573            result = store.select('df',                              1573            result = store.select('df',
1574                                  "string='foo' and string2='foo'"   1574                                  "string='foo' and string2='foo'"
1575                                  " and A>0 and B<0")                1575                                  " and A>0 and B<0")
1576            expected = df_new[(df_new.string == 'foo') & (           1576            expected = df_new[(df_new.string == 'foo') & (
1577                df_new.string2 == 'foo') & (df_new.A > 0) & (df_new.1577                df_new.string2 == 'foo') & (df_new.A > 0) & (df_new.
                                                                 B < 0)]                                                                 B < 0)]
1578            tm.assert_frame_equal(result, expected, check_index_type1578            tm.assert_frame_equal(result, expected, check_index_type
                                                                 =False)                                                                 =False)
1579                                                                     1579
1580            # yield an empty frame                                   1580            # yield an empty frame
1581            result = store.select('df', "string='foo' and string2='c1581            result = store.select('df', "string='foo' and string2='c
                                                                  ool'")                                                                  ool'")
1582            expected = df_new[(df_new.string == 'foo') & (           1582            expected = df_new[(df_new.string == 'foo') & (
1583                df_new.string2 == 'cool')]                           1583                df_new.string2 == 'cool')]
1584            tm.assert_frame_equal(result, expected, check_index_type1584            tm.assert_frame_equal(result, expected, check_index_type
                                                                 =False)                                                                 =False)
1585                                                                     1585
1586        with ensure_clean_store(self.path) as store:                 1586        with ensure_clean_store(self.path) as store:
1587            # doc example                                            1587            # doc example
1588            df_dc = df.copy()                                        1588            df_dc = df.copy()
1589            df_dc['string'] = 'foo'                                  1589            df_dc['string'] = 'foo'
1590            df_dc.loc[4:6, 'string'] = np.nan                        1590            df_dc.loc[4:6, 'string'] = np.nan
1591            df_dc.loc[7:9, 'string'] = 'bar'                         1591            df_dc.loc[7:9, 'string'] = 'bar'
1592            df_dc['string2'] = 'cool'                                1592            df_dc['string2'] = 'cool'
1593            df_dc['datetime'] = Timestamp('20010102')                1593            df_dc['datetime'] = Timestamp('20010102')
1594            df_dc = df_dc._convert(datetime=True)                    1594            df_dc = df_dc._convert(datetime=True)
1595            df_dc.loc[3:5, ['A', 'B', 'datetime']] = np.nan          1595            df_dc.loc[3:5, ['A', 'B', 'datetime']] = np.nan
1596                                                                     1596
1597            _maybe_remove(store, 'df_dc')                            1597            _maybe_remove(store, 'df_dc')
1598            store.append('df_dc', df_dc,                             1598            store.append('df_dc', df_dc,
1599                         data_columns=['B', 'C', 'string',           1599                         data_columns=['B', 'C', 'string',
1600                                       'string2', 'datetime'])       1600                                       'string2', 'datetime'])
1601            result = store.select('df_dc', 'B>0')                    1601            result = store.select('df_dc', 'B>0')
1602                                                                     1602
1603            expected = df_dc[df_dc.B > 0]                            1603            expected = df_dc[df_dc.B > 0]
1604            tm.assert_frame_equal(result, expected, check_index_type1604            tm.assert_frame_equal(result, expected, check_index_type
                                                                 =False)                                                                 =False)
1605                                                                     1605
1606            result = store.select(                                   1606            result = store.select(
1607                'df_dc', ['B > 0', 'C > 0', 'string == foo'])        1607                'df_dc', ['B > 0', 'C > 0', 'string == foo'])
1608            expected = df_dc[(df_dc.B > 0) & (df_dc.C > 0) & (       1608            expected = df_dc[(df_dc.B > 0) & (df_dc.C > 0) & (
1609                df_dc.string == 'foo')]                              1609                df_dc.string == 'foo')]
1610            tm.assert_frame_equal(result, expected, check_index_type1610            tm.assert_frame_equal(result, expected, check_index_type
                                                                 =False)                                                                 =False)
1611                                                                     1611
1612        with ensure_clean_store(self.path) as store:                 1612        with ensure_clean_store(self.path) as store:
1613            # doc example part 2                                     1613            # doc example part 2
1614            np.random.seed(1234)                                     1614            np.random.seed(1234)
1615            index = date_range('1/1/2000', periods=8)                1615            index = date_range('1/1/2000', periods=8)
1616            df_dc = DataFrame(np.random.randn(8, 3), index=index,    1616            df_dc = DataFrame(np.random.randn(8, 3), index=index,
1617                              columns=['A', 'B', 'C'])               1617                              columns=['A', 'B', 'C'])
1618            df_dc['string'] = 'foo'                                  1618            df_dc['string'] = 'foo'
1619            df_dc.loc[4:6, 'string'] = np.nan                        1619            df_dc.loc[4:6, 'string'] = np.nan
1620            df_dc.loc[7:9, 'string'] = 'bar'                         1620            df_dc.loc[7:9, 'string'] = 'bar'
1621            df_dc.loc[:, ['B', 'C']] = df_dc.loc[:, ['B', 'C']].abs()1621            df_dc.loc[:, ['B', 'C']] = df_dc.loc[:, ['B', 'C']].abs()
1622            df_dc['string2'] = 'cool'                                1622            df_dc['string2'] = 'cool'
1623                                                                     1623
1624            # on-disk operations                                     1624            # on-disk operations
1625            store.append('df_dc', df_dc, data_columns=[              1625            store.append('df_dc', df_dc, data_columns=[
1626                         'B', 'C', 'string', 'string2'])             1626                         'B', 'C', 'string', 'string2'])
1627                                                                     1627
1628            result = store.select('df_dc', 'B>0')                    1628            result = store.select('df_dc', 'B>0')
1629            expected = df_dc[df_dc.B > 0]                            1629            expected = df_dc[df_dc.B > 0]
1630            tm.assert_frame_equal(result, expected)                  1630            tm.assert_frame_equal(result, expected)
1631                                                                     1631
1632            result = store.select(                                   1632            result = store.select(
1633                'df_dc', ['B > 0', 'C > 0', 'string == "foo"'])      1633                'df_dc', ['B > 0', 'C > 0', 'string == "foo"'])
1634            expected = df_dc[(df_dc.B > 0) & (df_dc.C > 0) &         1634            expected = df_dc[(df_dc.B > 0) & (df_dc.C > 0) &
1635                             (df_dc.string == 'foo')]                1635                             (df_dc.string == 'foo')]
1636            tm.assert_frame_equal(result, expected)                  1636            tm.assert_frame_equal(result, expected)
1637                                                                     1637
1638        with ensure_clean_store(self.path) as store:                 1638        with ensure_clean_store(self.path) as store:
1639            with catch_warnings(record=True):                        1639            with catch_warnings(record=True):
1640                # panel                                              1640                # panel
1641                # GH5717 not handling data_columns                   1641                # GH5717 not handling data_columns
1642                np.random.seed(1234)                                 1642                np.random.seed(1234)
1643                p = tm.makePanel()                                   1643                p = tm.makePanel()
1644                                                                     1644
1645                store.append('p1', p)                                1645                store.append('p1', p)
1646                tm.assert_panel_equal(store.select('p1'), p)         1646                tm.assert_panel_equal(store.select('p1'), p)
1647                                                                     1647
1648                store.append('p2', p, data_columns=True)             1648                store.append('p2', p, data_columns=True)
1649                tm.assert_panel_equal(store.select('p2'), p)         1649                tm.assert_panel_equal(store.select('p2'), p)
1650                                                                     1650
1651                result = store.select('p2', where='ItemA>0')         1651                result = store.select('p2', where='ItemA>0')
1652                expected = p.to_frame()                              1652                expected = p.to_frame()
1653                expected = expected[expected['ItemA'] > 0]           1653                expected = expected[expected['ItemA'] > 0]
1654                tm.assert_frame_equal(result.to_frame(), expected)   1654                tm.assert_frame_equal(result.to_frame(), expected)
1655                                                                     1655
1656                result = store.select(                               1656                result = store.select(
1657                    'p2', where='ItemA>0 & minor_axis=["A","B"]')    1657                    'p2', where='ItemA>0 & minor_axis=["A","B"]')
1658                expected = p.to_frame()                              1658                expected = p.to_frame()
1659                expected = expected[expected['ItemA'] > 0]           1659                expected = expected[expected['ItemA'] > 0]
1660                expected = expected[expected.reset_index(            1660                expected = expected[expected.reset_index(
1661                    level=['major']).index.isin(['A', 'B'])]         1661                    level=['major']).index.isin(['A', 'B'])]
1662                tm.assert_frame_equal(result.to_frame(), expected)   1662                tm.assert_frame_equal(result.to_frame(), expected)
1663                                                                     1663
1664    def test_create_table_index(self):                               1664    def test_create_table_index(self):
1665                                                                     1665
1666        with ensure_clean_store(self.path) as store:                 1666        with ensure_clean_store(self.path) as store:
1667                                                                     1667
1668            with catch_warnings(record=True):                        1668            with catch_warnings(record=True):
1669                def col(t, column):                                  1669                def col(t, column):
1670                    return getattr(store.get_storer(t).table.cols, c1670                    return getattr(store.get_storer(t).table.cols, c
                                                                  olumn)                                                                  olumn)
1671                                                                     1671
1672                # index=False                                        1672                # index=False
1673                wp = tm.makePanel()                                  1673                wp = tm.makePanel()
1674                store.append('p5', wp, index=False)                  1674                store.append('p5', wp, index=False)
1675                store.create_table_index('p5', columns=['major_axis'1675                store.create_table_index('p5', columns=['major_axis'
                                                                      ])                                                                      ])
1676                assert(col('p5', 'major_axis').is_indexed is True)   1676                assert(col('p5', 'major_axis').is_indexed is True)
1677                assert(col('p5', 'minor_axis').is_indexed is False)  1677                assert(col('p5', 'minor_axis').is_indexed is False)
1678                                                                     1678
1679                # index=True                                         1679                # index=True
1680                store.append('p5i', wp, index=True)                  1680                store.append('p5i', wp, index=True)
1681                assert(col('p5i', 'major_axis').is_indexed is True)  1681                assert(col('p5i', 'major_axis').is_indexed is True)
1682                assert(col('p5i', 'minor_axis').is_indexed is True)  1682                assert(col('p5i', 'minor_axis').is_indexed is True)
1683                                                                     1683
1684                # default optlevels                                  1684                # default optlevels
1685                store.get_storer('p5').create_index()                1685                store.get_storer('p5').create_index()
1686                assert(col('p5', 'major_axis').index.optlevel == 6)  1686                assert(col('p5', 'major_axis').index.optlevel == 6)
1687                assert(col('p5', 'minor_axis').index.kind == 'medium1687                assert(col('p5', 'minor_axis').index.kind == 'medium
                                                                      ')                                                                      ')
1688                                                                     1688
1689                # let's change the indexing scheme                   1689                # let's change the indexing scheme
1690                store.create_table_index('p5')                       1690                store.create_table_index('p5')
1691                assert(col('p5', 'major_axis').index.optlevel == 6)  1691                assert(col('p5', 'major_axis').index.optlevel == 6)
1692                assert(col('p5', 'minor_axis').index.kind == 'medium1692                assert(col('p5', 'minor_axis').index.kind == 'medium
                                                                      ')                                                                      ')
1693                store.create_table_index('p5', optlevel=9)           1693                store.create_table_index('p5', optlevel=9)
1694                assert(col('p5', 'major_axis').index.optlevel == 9)  1694                assert(col('p5', 'major_axis').index.optlevel == 9)
1695                assert(col('p5', 'minor_axis').index.kind == 'medium1695                assert(col('p5', 'minor_axis').index.kind == 'medium
                                                                      ')                                                                      ')
1696                store.create_table_index('p5', kind='full')          1696                store.create_table_index('p5', kind='full')
1697                assert(col('p5', 'major_axis').index.optlevel == 9)  1697                assert(col('p5', 'major_axis').index.optlevel == 9)
1698                assert(col('p5', 'minor_axis').index.kind == 'full') 1698                assert(col('p5', 'minor_axis').index.kind == 'full')
1699                store.create_table_index('p5', optlevel=1, kind='lig1699                store.create_table_index('p5', optlevel=1, kind='lig
                                                                    ht')                                                                    ht')
1700                assert(col('p5', 'major_axis').index.optlevel == 1)  1700                assert(col('p5', 'major_axis').index.optlevel == 1)
1701                assert(col('p5', 'minor_axis').index.kind == 'light')1701                assert(col('p5', 'minor_axis').index.kind == 'light')
1702                                                                     1702
1703                # data columns                                       1703                # data columns
1704                df = tm.makeTimeDataFrame()                          1704                df = tm.makeTimeDataFrame()
1705                df['string'] = 'foo'                                 1705                df['string'] = 'foo'
1706                df['string2'] = 'bar'                                1706                df['string2'] = 'bar'
1707                store.append('f', df, data_columns=['string', 'strin1707                store.append('f', df, data_columns=['string', 'strin
                                                                   g2'])                                                                   g2'])
1708                assert(col('f', 'index').is_indexed is True)         1708                assert(col('f', 'index').is_indexed is True)
1709                assert(col('f', 'string').is_indexed is True)        1709                assert(col('f', 'string').is_indexed is True)
1710                assert(col('f', 'string2').is_indexed is True)       1710                assert(col('f', 'string2').is_indexed is True)
1711                                                                     1711
1712                # specify index=columns                              1712                # specify index=columns
1713                store.append(                                        1713                store.append(
1714                    'f2', df, index=['string'],                      1714                    'f2', df, index=['string'],
1715                    data_columns=['string', 'string2'])              1715                    data_columns=['string', 'string2'])
1716                assert(col('f2', 'index').is_indexed is False)       1716                assert(col('f2', 'index').is_indexed is False)
1717                assert(col('f2', 'string').is_indexed is True)       1717                assert(col('f2', 'string').is_indexed is True)
1718                assert(col('f2', 'string2').is_indexed is False)     1718                assert(col('f2', 'string2').is_indexed is False)
1719                                                                     1719
1720                # try to index a non-table                           1720                # try to index a non-table
1721                _maybe_remove(store, 'f2')                           1721                _maybe_remove(store, 'f2')
1722                store.put('f2', df)                                  1722                store.put('f2', df)
1723                pytest.raises(TypeError, store.create_table_index, '1723                pytest.raises(TypeError, store.create_table_index, '
                                                                    f2')                                                                    f2')
1724                                                                     1724
1725    def test_append_diff_item_order(self):                           1725    def test_append_diff_item_order(self):
1726                                                                     1726
1727        with catch_warnings(record=True):                            1727        with catch_warnings(record=True):
1728            wp = tm.makePanel()                                      1728            wp = tm.makePanel()
1729            wp1 = wp.iloc[:, :10, :]                                 1729            wp1 = wp.iloc[:, :10, :]
1730            wp2 = wp.iloc[wp.items.get_indexer(['ItemC', 'ItemB', 'I1730            wp2 = wp.iloc[wp.items.get_indexer(['ItemC', 'ItemB', 'I
                                                                temA']),                                                                temA']),
1731                          10:, :]                                    1731                          10:, :]
1732                                                                     1732
1733            with ensure_clean_store(self.path) as store:             1733            with ensure_clean_store(self.path) as store:
1734                store.put('panel', wp1, format='table')              1734                store.put('panel', wp1, format='table')
1735                pytest.raises(ValueError, store.put, 'panel', wp2,   1735                pytest.raises(ValueError, store.put, 'panel', wp2,
1736                              append=True)                           1736                              append=True)
1737                                                                     1737
1738    def test_append_hierarchical(self):                              1738    def test_append_hierarchical(self):
1739        index = MultiIndex(levels=[['foo', 'bar', 'baz', 'qux'],     1739        index = MultiIndex(levels=[['foo', 'bar', 'baz', 'qux'],
1740                                   ['one', 'two', 'three']],         1740                                   ['one', 'two', 'three']],
1741                           labels=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3],   1741                           labels=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3],
1742                                   [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],  1742                                   [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],
1743                           names=['foo', 'bar'])                     1743                           names=['foo', 'bar'])
1744        df = DataFrame(np.random.randn(10, 3), index=index,          1744        df = DataFrame(np.random.randn(10, 3), index=index,
1745                       columns=['A', 'B', 'C'])                      1745                       columns=['A', 'B', 'C'])
1746                                                                     1746
1747        with ensure_clean_store(self.path) as store:                 1747        with ensure_clean_store(self.path) as store:
1748            store.append('mi', df)                                   1748            store.append('mi', df)
1749            result = store.select('mi')                              1749            result = store.select('mi')
1750            tm.assert_frame_equal(result, df)                        1750            tm.assert_frame_equal(result, df)
1751                                                                     1751
1752            # GH 3748                                                1752            # GH 3748
1753            result = store.select('mi', columns=['A', 'B'])          1753            result = store.select('mi', columns=['A', 'B'])
1754            expected = df.reindex(columns=['A', 'B'])                1754            expected = df.reindex(columns=['A', 'B'])
1755            tm.assert_frame_equal(result, expected)                  1755            tm.assert_frame_equal(result, expected)
1756                                                                     1756
1757        with ensure_clean_path('test.hdf') as path:                  1757        with ensure_clean_path('test.hdf') as path:
1758            df.to_hdf(path, 'df', format='table')                    1758            df.to_hdf(path, 'df', format='table')
1759            result = read_hdf(path, 'df', columns=['A', 'B'])        1759            result = read_hdf(path, 'df', columns=['A', 'B'])
1760            expected = df.reindex(columns=['A', 'B'])                1760            expected = df.reindex(columns=['A', 'B'])
1761            tm.assert_frame_equal(result, expected)                  1761            tm.assert_frame_equal(result, expected)
1762                                                                     1762
1763    def test_column_multiindex(self):                                1763    def test_column_multiindex(self):
1764        # GH 4710                                                    1764        # GH 4710
1765        # recreate multi-indexes properly                            1765        # recreate multi-indexes properly
1766                                                                     1766
1767        index = MultiIndex.from_tuples([('A', 'a'), ('A', 'b'),      1767        index = MultiIndex.from_tuples([('A', 'a'), ('A', 'b'),
1768                                        ('B', 'a'), ('B', 'b')],     1768                                        ('B', 'a'), ('B', 'b')],
1769                                       names=['first', 'second'])    1769                                       names=['first', 'second'])
1770        df = DataFrame(np.arange(12).reshape(3, 4), columns=index)   1770        df = DataFrame(np.arange(12).reshape(3, 4), columns=index)
1771        expected = df.copy()                                         1771        expected = df.copy()
1772        if isinstance(expected.index, RangeIndex):                   1772        if isinstance(expected.index, RangeIndex):
1773            expected.index = Int64Index(expected.index)              1773            expected.index = Int64Index(expected.index)
1774                                                                     1774
1775        with ensure_clean_store(self.path) as store:                 1775        with ensure_clean_store(self.path) as store:
1776                                                                     1776
1777            store.put('df', df)                                      1777            store.put('df', df)
1778            tm.assert_frame_equal(store['df'], expected,             1778            tm.assert_frame_equal(store['df'], expected,
1779                                  check_index_type=True,             1779                                  check_index_type=True,
1780                                  check_column_type=True)            1780                                  check_column_type=True)
1781                                                                     1781
1782            store.put('df1', df, format='table')                     1782            store.put('df1', df, format='table')
1783            tm.assert_frame_equal(store['df1'], expected,            1783            tm.assert_frame_equal(store['df1'], expected,
1784                                  check_index_type=True,             1784                                  check_index_type=True,
1785                                  check_column_type=True)            1785                                  check_column_type=True)
1786                                                                     1786
1787            pytest.raises(ValueError, store.put, 'df2', df,          1787            pytest.raises(ValueError, store.put, 'df2', df,
1788                          format='table', data_columns=['A'])        1788                          format='table', data_columns=['A'])
1789            pytest.raises(ValueError, store.put, 'df3', df,          1789            pytest.raises(ValueError, store.put, 'df3', df,
1790                          format='table', data_columns=True)         1790                          format='table', data_columns=True)
1791                                                                     1791
1792        # appending multi-column on existing table (see GH 6167)     1792        # appending multi-column on existing table (see GH 6167)
1793        with ensure_clean_store(self.path) as store:                 1793        with ensure_clean_store(self.path) as store:
1794            store.append('df2', df)                                  1794            store.append('df2', df)
1795            store.append('df2', df)                                  1795            store.append('df2', df)
1796                                                                     1796
1797            tm.assert_frame_equal(store['df2'], concat((df, df)))    1797            tm.assert_frame_equal(store['df2'], concat((df, df)))
1798                                                                     1798
1799        # non_index_axes name                                        1799        # non_index_axes name
1800        df = DataFrame(np.arange(12).reshape(3, 4),                  1800        df = DataFrame(np.arange(12).reshape(3, 4),
1801                       columns=Index(list('ABCD'), name='foo'))      1801                       columns=Index(list('ABCD'), name='foo'))
1802        expected = df.copy()                                         1802        expected = df.copy()
1803        if isinstance(expected.index, RangeIndex):                   1803        if isinstance(expected.index, RangeIndex):
1804            expected.index = Int64Index(expected.index)              1804            expected.index = Int64Index(expected.index)
1805                                                                     1805
1806        with ensure_clean_store(self.path) as store:                 1806        with ensure_clean_store(self.path) as store:
1807                                                                     1807
1808            store.put('df1', df, format='table')                     1808            store.put('df1', df, format='table')
1809            tm.assert_frame_equal(store['df1'], expected,            1809            tm.assert_frame_equal(store['df1'], expected,
1810                                  check_index_type=True,             1810                                  check_index_type=True,
1811                                  check_column_type=True)            1811                                  check_column_type=True)
1812                                                                     1812
1813    def test_store_multiindex(self):                                 1813    def test_store_multiindex(self):
1814                                                                     1814
1815        # validate multi-index names                                 1815        # validate multi-index names
1816        # GH 5527                                                    1816        # GH 5527
1817        with ensure_clean_store(self.path) as store:                 1817        with ensure_clean_store(self.path) as store:
1818                                                                     1818
1819            def make_index(names=None):                              1819            def make_index(names=None):
1820                return MultiIndex.from_tuples([(datetime.datetime(201820                return MultiIndex.from_tuples([(datetime.datetime(20
                                                             13, 12, d),                                                             13, 12, d),
1821                                                s, t)                1821                                                s, t)
1822                                               for d in range(1, 3)  1822                                               for d in range(1, 3)
1823                                               for s in range(2)     1823                                               for s in range(2)
1824                                               for t in range(3)],   1824                                               for t in range(3)],
1825                                              names=names)           1825                                              names=names)
1826                                                                     1826
1827            # no names                                               1827            # no names
1828            _maybe_remove(store, 'df')                               1828            _maybe_remove(store, 'df')
1829            df = DataFrame(np.zeros((12, 2)), columns=[              1829            df = DataFrame(np.zeros((12, 2)), columns=[
1830                           'a', 'b'], index=make_index())            1830                           'a', 'b'], index=make_index())
1831            store.append('df', df)                                   1831            store.append('df', df)
1832            tm.assert_frame_equal(store.select('df'), df)            1832            tm.assert_frame_equal(store.select('df'), df)
1833                                                                     1833
1834            # partial names                                          1834            # partial names
1835            _maybe_remove(store, 'df')                               1835            _maybe_remove(store, 'df')
1836            df = DataFrame(np.zeros((12, 2)), columns=[              1836            df = DataFrame(np.zeros((12, 2)), columns=[
1837                           'a', 'b'], index=make_index(['date', None1837                           'a', 'b'], index=make_index(['date', None
                                                               , None]))                                                               , None]))
1838            store.append('df', df)                                   1838            store.append('df', df)
1839            tm.assert_frame_equal(store.select('df'), df)            1839            tm.assert_frame_equal(store.select('df'), df)
1840                                                                     1840
1841            # series                                                 1841            # series
1842            _maybe_remove(store, 's')                                1842            _maybe_remove(store, 's')
1843            s = Series(np.zeros(12), index=make_index(['date', None,1843            s = Series(np.zeros(12), index=make_index(['date', None,
                                                                 None]))                                                                 None]))
1844            store.append('s', s)                                     1844            store.append('s', s)
1845            xp = Series(np.zeros(12), index=make_index(              1845            xp = Series(np.zeros(12), index=make_index(
1846                ['date', 'level_1', 'level_2']))                     1846                ['date', 'level_1', 'level_2']))
1847            tm.assert_series_equal(store.select('s'), xp)            1847            tm.assert_series_equal(store.select('s'), xp)
1848                                                                     1848
1849            # dup with column                                        1849            # dup with column
1850            _maybe_remove(store, 'df')                               1850            _maybe_remove(store, 'df')
1851            df = DataFrame(np.zeros((12, 2)), columns=[              1851            df = DataFrame(np.zeros((12, 2)), columns=[
1852                           'a', 'b'], index=make_index(['date', 'a',1852                           'a', 'b'], index=make_index(['date', 'a',
                                                                  't']))                                                                  't']))
1853            pytest.raises(ValueError, store.append, 'df', df)        1853            pytest.raises(ValueError, store.append, 'df', df)
1854                                                                     1854
1855            # dup within level                                       1855            # dup within level
1856            _maybe_remove(store, 'df')                               1856            _maybe_remove(store, 'df')
1857            df = DataFrame(np.zeros((12, 2)), columns=['a', 'b'],    1857            df = DataFrame(np.zeros((12, 2)), columns=['a', 'b'],
1858                           index=make_index(['date', 'date', 'date']1858                           index=make_index(['date', 'date', 'date']
                                                                      ))                                                                      ))
1859            pytest.raises(ValueError, store.append, 'df', df)        1859            pytest.raises(ValueError, store.append, 'df', df)
1860                                                                     1860
1861            # fully names                                            1861            # fully names
1862            _maybe_remove(store, 'df')                               1862            _maybe_remove(store, 'df')
1863            df = DataFrame(np.zeros((12, 2)), columns=[              1863            df = DataFrame(np.zeros((12, 2)), columns=[
1864                           'a', 'b'], index=make_index(['date', 's',1864                           'a', 'b'], index=make_index(['date', 's',
                                                                  't']))                                                                  't']))
1865            store.append('df', df)                                   1865            store.append('df', df)
1866            tm.assert_frame_equal(store.select('df'), df)            1866            tm.assert_frame_equal(store.select('df'), df)
1867                                                                     1867
1868    def test_select_columns_in_where(self):                          1868    def test_select_columns_in_where(self):
1869                                                                     1869
1870        # GH 6169                                                    1870        # GH 6169
1871        # recreate multi-indexes when columns is passed              1871        # recreate multi-indexes when columns is passed
1872        # in the `where` argument                                    1872        # in the `where` argument
1873        index = MultiIndex(levels=[['foo', 'bar', 'baz', 'qux'],     1873        index = MultiIndex(levels=[['foo', 'bar', 'baz', 'qux'],
1874                                   ['one', 'two', 'three']],         1874                                   ['one', 'two', 'three']],
1875                           labels=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3],   1875                           labels=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3],
1876                                   [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],  1876                                   [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],
1877                           names=['foo_name', 'bar_name'])           1877                           names=['foo_name', 'bar_name'])
1878                                                                     1878
1879        # With a DataFrame                                           1879        # With a DataFrame
1880        df = DataFrame(np.random.randn(10, 3), index=index,          1880        df = DataFrame(np.random.randn(10, 3), index=index,
1881                       columns=['A', 'B', 'C'])                      1881                       columns=['A', 'B', 'C'])
1882                                                                     1882
1883        with ensure_clean_store(self.path) as store:                 1883        with ensure_clean_store(self.path) as store:
1884            store.put('df', df, format='table')                      1884            store.put('df', df, format='table')
1885            expected = df[['A']]                                     1885            expected = df[['A']]
1886                                                                     1886
1887            tm.assert_frame_equal(store.select('df', columns=['A']),1887            tm.assert_frame_equal(store.select('df', columns=['A']),
                                                               expected)                                                               expected)
1888                                                                     1888
1889            tm.assert_frame_equal(store.select(                      1889            tm.assert_frame_equal(store.select(
1890                'df', where="columns=['A']"), expected)              1890                'df', where="columns=['A']"), expected)
1891                                                                     1891
1892        # With a Series                                              1892        # With a Series
1893        s = Series(np.random.randn(10), index=index,                 1893        s = Series(np.random.randn(10), index=index,
1894                   name='A')                                         1894                   name='A')
1895        with ensure_clean_store(self.path) as store:                 1895        with ensure_clean_store(self.path) as store:
1896            store.put('s', s, format='table')                        1896            store.put('s', s, format='table')
1897            tm.assert_series_equal(store.select('s', where="columns=1897            tm.assert_series_equal(store.select('s', where="columns=
                                                             ['A']"), s)                                                             ['A']"), s)
1898                                                                     1898
1899    def test_mi_data_columns(self):                                  1899    def test_mi_data_columns(self):
1900        # GH 14435                                                   1900        # GH 14435
1901        idx = pd.MultiIndex.from_arrays([date_range('2000-01-01', pe1901        idx = pd.MultiIndex.from_arrays([date_range('2000-01-01', pe
                                                               riods=5),                                                               riods=5),
1902                                         range(5)], names=['date', '1902                                         range(5)], names=['date', '
                                                                   id'])                                                                   id'])
1903        df = pd.DataFrame({'a': [1.1, 1.2, 1.3, 1.4, 1.5]}, index=id1903        df = pd.DataFrame({'a': [1.1, 1.2, 1.3, 1.4, 1.5]}, index=id
                                                                      x)                                                                      x)
1904                                                                     1904
1905        with ensure_clean_store(self.path) as store:                 1905        with ensure_clean_store(self.path) as store:
1906            store.append('df', df, data_columns=True)                1906            store.append('df', df, data_columns=True)
1907                                                                     1907
1908            actual = store.select('df', where='id == 1')             1908            actual = store.select('df', where='id == 1')
1909            expected = df.iloc[[1], :]                               1909            expected = df.iloc[[1], :]
1910            tm.assert_frame_equal(actual, expected)                  1910            tm.assert_frame_equal(actual, expected)
1911                                                                     1911
1912    def test_pass_spec_to_storer(self):                              1912    def test_pass_spec_to_storer(self):
1913                                                                     1913
1914        df = tm.makeDataFrame()                                      1914        df = tm.makeDataFrame()
1915                                                                     1915
1916        with ensure_clean_store(self.path) as store:                 1916        with ensure_clean_store(self.path) as store:
1917            store.put('df', df)                                      1917            store.put('df', df)
1918            pytest.raises(TypeError, store.select, 'df', columns=['A1918            pytest.raises(TypeError, store.select, 'df', columns=['A
                                                                     '])                                                                     '])
1919            pytest.raises(TypeError, store.select,                   1919            pytest.raises(TypeError, store.select,
1920                          'df', where=[('columns=A')])               1920                          'df', where=[('columns=A')])
1921                                                                     1921
1922    def test_append_misc(self):                                      1922    def test_append_misc(self):
1923                                                                     1923
1924        with ensure_clean_store(self.path) as store:                 1924        with ensure_clean_store(self.path) as store:
1925                                                                     1925
1926            with catch_warnings(record=True):                        1926            with catch_warnings(record=True):
1927                                                                     1927
1928                # unsuported data types for non-tables               1928                # unsuported data types for non-tables
1929                p4d = tm.makePanel4D()                               1929                p4d = tm.makePanel4D()
1930                pytest.raises(TypeError, store.put, 'p4d', p4d)      1930                pytest.raises(TypeError, store.put, 'p4d', p4d)
1931                                                                     1931
1932                # unsuported data types                              1932                # unsuported data types
1933                pytest.raises(TypeError, store.put, 'abc', None)     1933                pytest.raises(TypeError, store.put, 'abc', None)
1934                pytest.raises(TypeError, store.put, 'abc', '123')    1934                pytest.raises(TypeError, store.put, 'abc', '123')
1935                pytest.raises(TypeError, store.put, 'abc', 123)      1935                pytest.raises(TypeError, store.put, 'abc', 123)
1936                pytest.raises(TypeError, store.put, 'abc', np.arange1936                pytest.raises(TypeError, store.put, 'abc', np.arange
                                                                    (5))                                                                    (5))
1937                                                                     1937
1938                df = tm.makeDataFrame()                              1938                df = tm.makeDataFrame()
1939                store.append('df', df, chunksize=1)                  1939                store.append('df', df, chunksize=1)
1940                result = store.select('df')                          1940                result = store.select('df')
1941                tm.assert_frame_equal(result, df)                    1941                tm.assert_frame_equal(result, df)
1942                                                                     1942
1943                store.append('df1', df, expectedrows=10)             1943                store.append('df1', df, expectedrows=10)
1944                result = store.select('df1')                         1944                result = store.select('df1')
1945                tm.assert_frame_equal(result, df)                    1945                tm.assert_frame_equal(result, df)
1946                                                                     1946
1947        # more chunksize in append tests                             1947        # more chunksize in append tests
1948        def check(obj, comparator):                                  1948        def check(obj, comparator):
1949            for c in [10, 200, 1000]:                                1949            for c in [10, 200, 1000]:
1950                with ensure_clean_store(self.path, mode='w') as stor1950                with ensure_clean_store(self.path, mode='w') as stor
                                                                      e:                                                                      e:
1951                    store.append('obj', obj, chunksize=c)            1951                    store.append('obj', obj, chunksize=c)
1952                    result = store.select('obj')                     1952                    result = store.select('obj')
1953                    comparator(result, obj)                          1953                    comparator(result, obj)
1954                                                                     1954
1955        df = tm.makeDataFrame()                                      1955        df = tm.makeDataFrame()
1956        df['string'] = 'foo'                                         1956        df['string'] = 'foo'
1957        df['float322'] = 1.                                          1957        df['float322'] = 1.
1958        df['float322'] = df['float322'].astype('float32')            1958        df['float322'] = df['float322'].astype('float32')
1959        df['bool'] = df['float322'] > 0                              1959        df['bool'] = df['float322'] > 0
1960        df['time1'] = Timestamp('20130101')                          1960        df['time1'] = Timestamp('20130101')
1961        df['time2'] = Timestamp('20130102')                          1961        df['time2'] = Timestamp('20130102')
1962        check(df, tm.assert_frame_equal)                             1962        check(df, tm.assert_frame_equal)
1963                                                                     1963
1964        with catch_warnings(record=True):                            1964        with catch_warnings(record=True):
1965            p = tm.makePanel()                                       1965            p = tm.makePanel()
1966            check(p, assert_panel_equal)                             1966            check(p, assert_panel_equal)
1967                                                                     1967
1968        with catch_warnings(record=True):                            1968        with catch_warnings(record=True):
1969            p4d = tm.makePanel4D()                                   1969            p4d = tm.makePanel4D()
1970            check(p4d, assert_panel4d_equal)                         1970            check(p4d, assert_panel4d_equal)
1971                                                                     1971
1972        # empty frame, GH4273                                        1972        # empty frame, GH4273
1973        with ensure_clean_store(self.path) as store:                 1973        with ensure_clean_store(self.path) as store:
1974                                                                     1974
1975            # 0 len                                                  1975            # 0 len
1976            df_empty = DataFrame(columns=list('ABC'))                1976            df_empty = DataFrame(columns=list('ABC'))
1977            store.append('df', df_empty)                             1977            store.append('df', df_empty)
1978            pytest.raises(KeyError, store.select, 'df')              1978            pytest.raises(KeyError, store.select, 'df')
1979                                                                     1979
1980            # repeated append of 0/non-zero frames                   1980            # repeated append of 0/non-zero frames
1981            df = DataFrame(np.random.rand(10, 3), columns=list('ABC'1981            df = DataFrame(np.random.rand(10, 3), columns=list('ABC'
                                                                      ))                                                                      ))
1982            store.append('df', df)                                   1982            store.append('df', df)
1983            assert_frame_equal(store.select('df'), df)               1983            assert_frame_equal(store.select('df'), df)
1984            store.append('df', df_empty)                             1984            store.append('df', df_empty)
1985            assert_frame_equal(store.select('df'), df)               1985            assert_frame_equal(store.select('df'), df)
1986                                                                     1986
1987            # store                                                  1987            # store
1988            df = DataFrame(columns=list('ABC'))                      1988            df = DataFrame(columns=list('ABC'))
1989            store.put('df2', df)                                     1989            store.put('df2', df)
1990            assert_frame_equal(store.select('df2'), df)              1990            assert_frame_equal(store.select('df2'), df)
1991                                                                     1991
1992            with catch_warnings(record=True):                        1992            with catch_warnings(record=True):
1993                                                                     1993
1994                # 0 len                                              1994                # 0 len
1995                p_empty = Panel(items=list('ABC'))                   1995                p_empty = Panel(items=list('ABC'))
1996                store.append('p', p_empty)                           1996                store.append('p', p_empty)
1997                pytest.raises(KeyError, store.select, 'p')           1997                pytest.raises(KeyError, store.select, 'p')
1998                                                                     1998
1999                # repeated append of 0/non-zero frames               1999                # repeated append of 0/non-zero frames
2000                p = Panel(np.random.randn(3, 4, 5), items=list('ABC'2000                p = Panel(np.random.randn(3, 4, 5), items=list('ABC'
                                                                      ))                                                                      ))
2001                store.append('p', p)                                 2001                store.append('p', p)
2002                assert_panel_equal(store.select('p'), p)             2002                assert_panel_equal(store.select('p'), p)
2003                store.append('p', p_empty)                           2003                store.append('p', p_empty)
2004                assert_panel_equal(store.select('p'), p)             2004                assert_panel_equal(store.select('p'), p)
2005                                                                     2005
2006                # store                                              2006                # store
2007                store.put('p2', p_empty)                             2007                store.put('p2', p_empty)
2008                assert_panel_equal(store.select('p2'), p_empty)      2008                assert_panel_equal(store.select('p2'), p_empty)
2009                                                                     2009
2010    def test_append_raise(self):                                     2010    def test_append_raise(self):
2011                                                                     2011
2012        with ensure_clean_store(self.path) as store:                 2012        with ensure_clean_store(self.path) as store:
2013                                                                     2013
2014            # test append with invalid input to get good error messa2014            # test append with invalid input to get good error messa
                                                                     ges                                                                     ges
2015                                                                     2015
2016            # list in column                                         2016            # list in column
2017            df = tm.makeDataFrame()                                  2017            df = tm.makeDataFrame()
2018            df['invalid'] = [['a']] * len(df)                        2018            df['invalid'] = [['a']] * len(df)
2019            assert df.dtypes['invalid'] == np.object_                2019            assert df.dtypes['invalid'] == np.object_
2020            pytest.raises(TypeError, store.append, 'df', df)         2020            pytest.raises(TypeError, store.append, 'df', df)
2021                                                                     2021
2022            # multiple invalid columns                               2022            # multiple invalid columns
2023            df['invalid2'] = [['a']] * len(df)                       2023            df['invalid2'] = [['a']] * len(df)
2024            df['invalid3'] = [['a']] * len(df)                       2024            df['invalid3'] = [['a']] * len(df)
2025            pytest.raises(TypeError, store.append, 'df', df)         2025            pytest.raises(TypeError, store.append, 'df', df)
2026                                                                     2026
2027            # datetime with embedded nans as object                  2027            # datetime with embedded nans as object
2028            df = tm.makeDataFrame()                                  2028            df = tm.makeDataFrame()
2029            s = Series(datetime.datetime(2001, 1, 2), index=df.index)2029            s = Series(datetime.datetime(2001, 1, 2), index=df.index)
2030            s = s.astype(object)                                     2030            s = s.astype(object)
2031            s[0:5] = np.nan                                          2031            s[0:5] = np.nan
2032            df['invalid'] = s                                        2032            df['invalid'] = s
2033            assert df.dtypes['invalid'] == np.object_                2033            assert df.dtypes['invalid'] == np.object_
2034            pytest.raises(TypeError, store.append, 'df', df)         2034            pytest.raises(TypeError, store.append, 'df', df)
2035                                                                     2035
2036            # directy ndarray                                        2036            # directy ndarray
2037            pytest.raises(TypeError, store.append, 'df', np.arange(12037            pytest.raises(TypeError, store.append, 'df', np.arange(1
                                                                     0))                                                                     0))
2038                                                                     2038
2039            # series directly                                        2039            # series directly
2040            pytest.raises(TypeError, store.append,                   2040            pytest.raises(TypeError, store.append,
2041                          'df', Series(np.arange(10)))               2041                          'df', Series(np.arange(10)))
2042                                                                     2042
2043            # appending an incompatible table                        2043            # appending an incompatible table
2044            df = tm.makeDataFrame()                                  2044            df = tm.makeDataFrame()
2045            store.append('df', df)                                   2045            store.append('df', df)
2046                                                                     2046
2047            df['foo'] = 'foo'                                        2047            df['foo'] = 'foo'
2048            pytest.raises(ValueError, store.append, 'df', df)        2048            pytest.raises(ValueError, store.append, 'df', df)
2049                                                                     2049
2050    def test_table_index_incompatible_dtypes(self):                  2050    def test_table_index_incompatible_dtypes(self):
2051        df1 = DataFrame({'a': [1, 2, 3]})                            2051        df1 = DataFrame({'a': [1, 2, 3]})
2052        df2 = DataFrame({'a': [4, 5, 6]},                            2052        df2 = DataFrame({'a': [4, 5, 6]},
2053                        index=date_range('1/1/2000', periods=3))     2053                        index=date_range('1/1/2000', periods=3))
2054                                                                     2054
2055        with ensure_clean_store(self.path) as store:                 2055        with ensure_clean_store(self.path) as store:
2056            store.put('frame', df1, format='table')                  2056            store.put('frame', df1, format='table')
2057            pytest.raises(TypeError, store.put, 'frame', df2,        2057            pytest.raises(TypeError, store.put, 'frame', df2,
2058                          format='table', append=True)               2058                          format='table', append=True)
2059                                                                     2059
2060    def test_table_values_dtypes_roundtrip(self):                    2060    def test_table_values_dtypes_roundtrip(self):
2061                                                                     2061
2062        with ensure_clean_store(self.path) as store:                 2062        with ensure_clean_store(self.path) as store:
2063            df1 = DataFrame({'a': [1, 2, 3]}, dtype='f8')            2063            df1 = DataFrame({'a': [1, 2, 3]}, dtype='f8')
2064            store.append('df_f8', df1)                               2064            store.append('df_f8', df1)
2065            assert_series_equal(df1.dtypes, store['df_f8'].dtypes)   2065            assert_series_equal(df1.dtypes, store['df_f8'].dtypes)
2066                                                                     2066
2067            df2 = DataFrame({'a': [1, 2, 3]}, dtype='i8')            2067            df2 = DataFrame({'a': [1, 2, 3]}, dtype='i8')
2068            store.append('df_i8', df2)                               2068            store.append('df_i8', df2)
2069            assert_series_equal(df2.dtypes, store['df_i8'].dtypes)   2069            assert_series_equal(df2.dtypes, store['df_i8'].dtypes)
2070                                                                     2070
2071            # incompatible dtype                                     2071            # incompatible dtype
2072            pytest.raises(ValueError, store.append, 'df_i8', df1)    2072            pytest.raises(ValueError, store.append, 'df_i8', df1)
2073                                                                     2073
2074            # check creation/storage/retrieval of float32 (a bit hac2074            # check creation/storage/retrieval of float32 (a bit hac
                                                                   ky to                                                                   ky to
2075            # actually create them thought)                          2075            # actually create them thought)
2076            df1 = DataFrame(                                         2076            df1 = DataFrame(
2077                np.array([[1], [2], [3]], dtype='f4'), columns=['A'])2077                np.array([[1], [2], [3]], dtype='f4'), columns=['A'])
2078            store.append('df_f4', df1)                               2078            store.append('df_f4', df1)
2079            assert_series_equal(df1.dtypes, store['df_f4'].dtypes)   2079            assert_series_equal(df1.dtypes, store['df_f4'].dtypes)
2080            assert df1.dtypes[0] == 'float32'                        2080            assert df1.dtypes[0] == 'float32'
2081                                                                     2081
2082            # check with mixed dtypes                                2082            # check with mixed dtypes
2083            df1 = DataFrame(dict([(c, Series(np.random.randn(5), dty2083            df1 = DataFrame(dict([(c, Series(np.random.randn(5), dty
                                                                  pe=c))                                                                  pe=c))
2084                                  for c in ['float32', 'float64', 'i2084                                  for c in ['float32', 'float64', 'i
                                                                  nt32',                                                                  nt32',
2085                                            'int64', 'int16', 'int8'2085                                            'int64', 'int16', 'int8'
                                                                    ]]))                                                                    ]]))
2086            df1['string'] = 'foo'                                    2086            df1['string'] = 'foo'
2087            df1['float322'] = 1.                                     2087            df1['float322'] = 1.
2088            df1['float322'] = df1['float322'].astype('float32')      2088            df1['float322'] = df1['float322'].astype('float32')
2089            df1['bool'] = df1['float32'] > 0                         2089            df1['bool'] = df1['float32'] > 0
2090            df1['time1'] = Timestamp('20130101')                     2090            df1['time1'] = Timestamp('20130101')
2091            df1['time2'] = Timestamp('20130102')                     2091            df1['time2'] = Timestamp('20130102')
2092                                                                     2092
2093            store.append('df_mixed_dtypes1', df1)                    2093            store.append('df_mixed_dtypes1', df1)
2094            result = store.select('df_mixed_dtypes1').get_dtype_coun2094            result = store.select('df_mixed_dtypes1').get_dtype_coun
                                                                    ts()                                                                    ts()
2095            expected = Series({'float32': 2, 'float64': 1, 'int32': 2095            expected = Series({'float32': 2, 'float64': 1, 'int32': 
                                                                      1,                                                                      1,
2096                               'bool': 1, 'int16': 1, 'int8': 1,     2096                               'bool': 1, 'int16': 1, 'int8': 1,
2097                               'int64': 1, 'object': 1, 'datetime64[2097                               'int64': 1, 'object': 1, 'datetime64[
                                                               ns]': 2})                                                               ns]': 2})
2098            result = result.sort_index()                             2098            result = result.sort_index()
2099            result = expected.sort_index()                           2099            result = expected.sort_index()
2100            tm.assert_series_equal(result, expected)                 2100            tm.assert_series_equal(result, expected)
2101                                                                     2101
2102    def test_table_mixed_dtypes(self):                               2102    def test_table_mixed_dtypes(self):
2103                                                                     2103
2104        # frame                                                      2104        # frame
2105        df = tm.makeDataFrame()                                      2105        df = tm.makeDataFrame()
2106        df['obj1'] = 'foo'                                           2106        df['obj1'] = 'foo'
2107        df['obj2'] = 'bar'                                           2107        df['obj2'] = 'bar'
2108        df['bool1'] = df['A'] > 0                                    2108        df['bool1'] = df['A'] > 0
2109        df['bool2'] = df['B'] > 0                                    2109        df['bool2'] = df['B'] > 0
2110        df['bool3'] = True                                           2110        df['bool3'] = True
2111        df['int1'] = 1                                               2111        df['int1'] = 1
2112        df['int2'] = 2                                               2112        df['int2'] = 2
2113        df['timestamp1'] = Timestamp('20010102')                     2113        df['timestamp1'] = Timestamp('20010102')
2114        df['timestamp2'] = Timestamp('20010103')                     2114        df['timestamp2'] = Timestamp('20010103')
2115        df['datetime1'] = datetime.datetime(2001, 1, 2, 0, 0)        2115        df['datetime1'] = datetime.datetime(2001, 1, 2, 0, 0)
2116        df['datetime2'] = datetime.datetime(2001, 1, 3, 0, 0)        2116        df['datetime2'] = datetime.datetime(2001, 1, 3, 0, 0)
2117        df.loc[3:6, ['obj1']] = np.nan                               2117        df.loc[3:6, ['obj1']] = np.nan
2118        df = df._consolidate()._convert(datetime=True)               2118        df = df._consolidate()._convert(datetime=True)
2119                                                                     2119
2120        with ensure_clean_store(self.path) as store:                 2120        with ensure_clean_store(self.path) as store:
2121            store.append('df1_mixed', df)                            2121            store.append('df1_mixed', df)
2122            tm.assert_frame_equal(store.select('df1_mixed'), df)     2122            tm.assert_frame_equal(store.select('df1_mixed'), df)
2123                                                                     2123
2124        with catch_warnings(record=True):                            2124        with catch_warnings(record=True):
2125                                                                     2125
2126            # panel                                                  2126            # panel
2127            wp = tm.makePanel()                                      2127            wp = tm.makePanel()
2128            wp['obj1'] = 'foo'                                       2128            wp['obj1'] = 'foo'
2129            wp['obj2'] = 'bar'                                       2129            wp['obj2'] = 'bar'
2130            wp['bool1'] = wp['ItemA'] > 0                            2130            wp['bool1'] = wp['ItemA'] > 0
2131            wp['bool2'] = wp['ItemB'] > 0                            2131            wp['bool2'] = wp['ItemB'] > 0
2132            wp['int1'] = 1                                           2132            wp['int1'] = 1
2133            wp['int2'] = 2                                           2133            wp['int2'] = 2
2134            wp = wp._consolidate()                                   2134            wp = wp._consolidate()
2135                                                                     2135
2136        with catch_warnings(record=True):                            2136        with catch_warnings(record=True):
2137                                                                     2137
2138            with ensure_clean_store(self.path) as store:             2138            with ensure_clean_store(self.path) as store:
2139                store.append('p1_mixed', wp)                         2139                store.append('p1_mixed', wp)
2140                assert_panel_equal(store.select('p1_mixed'), wp)     2140                assert_panel_equal(store.select('p1_mixed'), wp)
2141                                                                     2141
2142        with catch_warnings(record=True):                            2142        with catch_warnings(record=True):
2143            # ndim                                                   2143            # ndim
2144            wp = tm.makePanel4D()                                    2144            wp = tm.makePanel4D()
2145            wp['obj1'] = 'foo'                                       2145            wp['obj1'] = 'foo'
2146            wp['obj2'] = 'bar'                                       2146            wp['obj2'] = 'bar'
2147            wp['bool1'] = wp['l1'] > 0                               2147            wp['bool1'] = wp['l1'] > 0
2148            wp['bool2'] = wp['l2'] > 0                               2148            wp['bool2'] = wp['l2'] > 0
2149            wp['int1'] = 1                                           2149            wp['int1'] = 1
2150            wp['int2'] = 2                                           2150            wp['int2'] = 2
2151            wp = wp._consolidate()                                   2151            wp = wp._consolidate()
2152                                                                     2152
2153            with ensure_clean_store(self.path) as store:             2153            with ensure_clean_store(self.path) as store:
2154                store.append('p4d_mixed', wp)                        2154                store.append('p4d_mixed', wp)
2155                assert_panel4d_equal(store.select('p4d_mixed'), wp)  2155                assert_panel4d_equal(store.select('p4d_mixed'), wp)
2156                                                                     2156
2157    def test_unimplemented_dtypes_table_columns(self):               2157    def test_unimplemented_dtypes_table_columns(self):
2158                                                                     2158
2159        with ensure_clean_store(self.path) as store:                 2159        with ensure_clean_store(self.path) as store:
2160                                                                     2160
2161            l = [('date', datetime.date(2001, 1, 2))]                2161            l = [('date', datetime.date(2001, 1, 2))]
2162                                                                     2162
2163            # py3 ok for unicode                                     2163            # py3 ok for unicode
2164            if not compat.PY3:                                       2164            if not compat.PY3:
2165                l.append(('unicode', u('\\u03c3')))                  2165                l.append(('unicode', u('\\u03c3')))
2166                                                                     2166
2167            # currently not supported dtypes ####                    2167            # currently not supported dtypes ####
2168            for n, f in l:                                           2168            for n, f in l:
2169                df = tm.makeDataFrame()                              2169                df = tm.makeDataFrame()
2170                df[n] = f                                            2170                df[n] = f
2171                pytest.raises(                                       2171                pytest.raises(
2172                    TypeError, store.append, 'df1_%s' % n, df)       2172                    TypeError, store.append, 'df1_%s' % n, df)
2173                                                                     2173
2174        # frame                                                      2174        # frame
2175        df = tm.makeDataFrame()                                      2175        df = tm.makeDataFrame()
2176        df['obj1'] = 'foo'                                           2176        df['obj1'] = 'foo'
2177        df['obj2'] = 'bar'                                           2177        df['obj2'] = 'bar'
2178        df['datetime1'] = datetime.date(2001, 1, 2)                  2178        df['datetime1'] = datetime.date(2001, 1, 2)
2179        df = df._consolidate()._convert(datetime=True)               2179        df = df._consolidate()._convert(datetime=True)
2180                                                                     2180
2181        with ensure_clean_store(self.path) as store:                 2181        with ensure_clean_store(self.path) as store:
2182            # this fails because we have a date in the object block.2182            # this fails because we have a date in the object block.
                                                                   .....                                                                   .....
2183            pytest.raises(TypeError, store.append, 'df_unimplemented2183            pytest.raises(TypeError, store.append, 'df_unimplemented
                                                                  ', df)                                                                  ', df)
2184                                                                     2184
2185    def test_calendar_roundtrip_issue(self):                         2185    def test_calendar_roundtrip_issue(self):
2186                                                                     2186
2187        # 8591                                                       2187        # 8591
2188        # doc example from tseries holiday section                   2188        # doc example from tseries holiday section
2189        weekmask_egypt = 'Sun Mon Tue Wed Thu'                       2189        weekmask_egypt = 'Sun Mon Tue Wed Thu'
2190        holidays = ['2012-05-01',                                    2190        holidays = ['2012-05-01',
2191                    datetime.datetime(2013, 5, 1), np.datetime64('202191                    datetime.datetime(2013, 5, 1), np.datetime64('20
                                                             14-05-01')]                                                             14-05-01')]
2192        bday_egypt = pandas.offsets.CustomBusinessDay(               2192        bday_egypt = pandas.offsets.CustomBusinessDay(
2193            holidays=holidays, weekmask=weekmask_egypt)              2193            holidays=holidays, weekmask=weekmask_egypt)
2194        dt = datetime.datetime(2013, 4, 30)                          2194        dt = datetime.datetime(2013, 4, 30)
2195        dts = date_range(dt, periods=5, freq=bday_egypt)             2195        dts = date_range(dt, periods=5, freq=bday_egypt)
2196                                                                     2196
2197        s = (Series(dts.weekday, dts).map(                           2197        s = (Series(dts.weekday, dts).map(
2198            Series('Mon Tue Wed Thu Fri Sat Sun'.split())))          2198            Series('Mon Tue Wed Thu Fri Sat Sun'.split())))
2199                                                                     2199
2200        with ensure_clean_store(self.path) as store:                 2200        with ensure_clean_store(self.path) as store:
2201                                                                     2201
2202            store.put('fixed', s)                                    2202            store.put('fixed', s)
2203            result = store.select('fixed')                           2203            result = store.select('fixed')
2204            assert_series_equal(result, s)                           2204            assert_series_equal(result, s)
2205                                                                     2205
2206            store.append('table', s)                                 2206            store.append('table', s)
2207            result = store.select('table')                           2207            result = store.select('table')
2208            assert_series_equal(result, s)                           2208            assert_series_equal(result, s)
2209                                                                     2209
2210    def test_append_with_timedelta(self):                            2210    def test_append_with_timedelta(self):
2211        # GH 3577                                                    2211        # GH 3577
2212        # append timedelta                                           2212        # append timedelta
2213                                                                     2213
2214        df = DataFrame(dict(A=Timestamp('20130101'), B=[Timestamp(   2214        df = DataFrame(dict(A=Timestamp('20130101'), B=[Timestamp(
2215            '20130101') + timedelta(days=i, seconds=10) for i in ran2215            '20130101') + timedelta(days=i, seconds=10) for i in ran
                                                               ge(10)]))                                                               ge(10)]))
2216        df['C'] = df['A'] - df['B']                                  2216        df['C'] = df['A'] - df['B']
2217        df.loc[3:5, 'C'] = np.nan                                    2217        df.loc[3:5, 'C'] = np.nan
2218                                                                     2218
2219        with ensure_clean_store(self.path) as store:                 2219        with ensure_clean_store(self.path) as store:
2220                                                                     2220
2221            # table                                                  2221            # table
2222            _maybe_remove(store, 'df')                               2222            _maybe_remove(store, 'df')
2223            store.append('df', df, data_columns=True)                2223            store.append('df', df, data_columns=True)
2224            result = store.select('df')                              2224            result = store.select('df')
2225            assert_frame_equal(result, df)                           2225            assert_frame_equal(result, df)
2226                                                                     2226
2227            result = store.select('df', where="C<100000")            2227            result = store.select('df', where="C<100000")
2228            assert_frame_equal(result, df)                           2228            assert_frame_equal(result, df)
2229                                                                     2229
2230            result = store.select('df', where="C<pd.Timedelta('-3D')2230            result = store.select('df', where="C<pd.Timedelta('-3D')
                                                                      ")                                                                      ")
2231            assert_frame_equal(result, df.iloc[3:])                  2231            assert_frame_equal(result, df.iloc[3:])
2232                                                                     2232
2233            result = store.select('df', "C<'-3D'")                   2233            result = store.select('df', "C<'-3D'")
2234            assert_frame_equal(result, df.iloc[3:])                  2234            assert_frame_equal(result, df.iloc[3:])
2235                                                                     2235
2236            # a bit hacky here as we don't really deal with the NaT 2236            # a bit hacky here as we don't really deal with the NaT 
                                                                properly                                                                properly
2237                                                                     2237
2238            result = store.select('df', "C<'-500000s'")              2238            result = store.select('df', "C<'-500000s'")
2239            result = result.dropna(subset=['C'])                     2239            result = result.dropna(subset=['C'])
2240            assert_frame_equal(result, df.iloc[6:])                  2240            assert_frame_equal(result, df.iloc[6:])
2241                                                                     2241
2242            result = store.select('df', "C<'-3.5D'")                 2242            result = store.select('df', "C<'-3.5D'")
2243            result = result.iloc[1:]                                 2243            result = result.iloc[1:]
2244            assert_frame_equal(result, df.iloc[4:])                  2244            assert_frame_equal(result, df.iloc[4:])
2245                                                                     2245
2246            # fixed                                                  2246            # fixed
2247            _maybe_remove(store, 'df2')                              2247            _maybe_remove(store, 'df2')
2248            store.put('df2', df)                                     2248            store.put('df2', df)
2249            result = store.select('df2')                             2249            result = store.select('df2')
2250            assert_frame_equal(result, df)                           2250            assert_frame_equal(result, df)
2251                                                                     2251
2252    def test_remove(self):                                           2252    def test_remove(self):
2253                                                                     2253
2254        with ensure_clean_store(self.path) as store:                 2254        with ensure_clean_store(self.path) as store:
2255                                                                     2255
2256            ts = tm.makeTimeSeries()                                 2256            ts = tm.makeTimeSeries()
2257            df = tm.makeDataFrame()                                  2257            df = tm.makeDataFrame()
2258            store['a'] = ts                                          2258            store['a'] = ts
2259            store['b'] = df                                          2259            store['b'] = df
2260            _maybe_remove(store, 'a')                                2260            _maybe_remove(store, 'a')
2261            assert len(store) == 1                                   2261            assert len(store) == 1
2262            tm.assert_frame_equal(df, store['b'])                    2262            tm.assert_frame_equal(df, store['b'])
2263                                                                     2263
2264            _maybe_remove(store, 'b')                                2264            _maybe_remove(store, 'b')
2265            assert len(store) == 0                                   2265            assert len(store) == 0
2266                                                                     2266
2267            # nonexistence                                           2267            # nonexistence
2268            pytest.raises(KeyError, store.remove, 'a_nonexistent_sto2268            pytest.raises(KeyError, store.remove, 'a_nonexistent_sto
                                                                    re')                                                                    re')
2269                                                                     2269
2270            # pathing                                                2270            # pathing
2271            store['a'] = ts                                          2271            store['a'] = ts
2272            store['b/foo'] = df                                      2272            store['b/foo'] = df
2273            _maybe_remove(store, 'foo')                              2273            _maybe_remove(store, 'foo')
2274            _maybe_remove(store, 'b/foo')                            2274            _maybe_remove(store, 'b/foo')
2275            assert len(store) == 1                                   2275            assert len(store) == 1
2276                                                                     2276
2277            store['a'] = ts                                          2277            store['a'] = ts
2278            store['b/foo'] = df                                      2278            store['b/foo'] = df
2279            _maybe_remove(store, 'b')                                2279            _maybe_remove(store, 'b')
2280            assert len(store) == 1                                   2280            assert len(store) == 1
2281                                                                     2281
2282            # __delitem__                                            2282            # __delitem__
2283            store['a'] = ts                                          2283            store['a'] = ts
2284            store['b'] = df                                          2284            store['b'] = df
2285            del store['a']                                           2285            del store['a']
2286            del store['b']                                           2286            del store['b']
2287            assert len(store) == 0                                   2287            assert len(store) == 0
2288                                                                     2288
2289    def test_remove_where(self):                                     2289    def test_remove_where(self):
2290                                                                     2290
2291        with ensure_clean_store(self.path) as store:                 2291        with ensure_clean_store(self.path) as store:
2292                                                                     2292
2293            with catch_warnings(record=True):                        2293            with catch_warnings(record=True):
2294                                                                     2294
2295                # non-existance                                      2295                # non-existance
2296                crit1 = 'index>foo'                                  2296                crit1 = 'index>foo'
2297                pytest.raises(KeyError, store.remove, 'a', [crit1])  2297                pytest.raises(KeyError, store.remove, 'a', [crit1])
2298                                                                     2298
2299                # try to remove non-table (with crit)                2299                # try to remove non-table (with crit)
2300                # non-table ok (where = None)                        2300                # non-table ok (where = None)
2301                wp = tm.makePanel(30)                                2301                wp = tm.makePanel(30)
2302                store.put('wp', wp, format='table')                  2302                store.put('wp', wp, format='table')
2303                store.remove('wp', ["minor_axis=['A', 'D']"])        2303                store.remove('wp', ["minor_axis=['A', 'D']"])
2304                rs = store.select('wp')                              2304                rs = store.select('wp')
2305                expected = wp.reindex(minor_axis=['B', 'C'])         2305                expected = wp.reindex(minor_axis=['B', 'C'])
2306                assert_panel_equal(rs, expected)                     2306                assert_panel_equal(rs, expected)
2307                                                                     2307
2308                # empty where                                        2308                # empty where
2309                _maybe_remove(store, 'wp')                           2309                _maybe_remove(store, 'wp')
2310                store.put('wp', wp, format='table')                  2310                store.put('wp', wp, format='table')
2311                                                                     2311
2312                # deleted number (entire table)                      2312                # deleted number (entire table)
2313                n = store.remove('wp', [])                           2313                n = store.remove('wp', [])
2314                assert n == 120                                      2314                assert n == 120
2315                                                                     2315
2316                # non - empty where                                  2316                # non - empty where
2317                _maybe_remove(store, 'wp')                           2317                _maybe_remove(store, 'wp')
2318                store.put('wp', wp, format='table')                  2318                store.put('wp', wp, format='table')
2319                pytest.raises(ValueError, store.remove,              2319                pytest.raises(ValueError, store.remove,
2320                              'wp', ['foo'])                         2320                              'wp', ['foo'])
2321                                                                     2321
2322    def test_remove_startstop(self):                                 2322    def test_remove_startstop(self):
2323        # GH #4835 and #6177                                         2323        # GH #4835 and #6177
2324                                                                     2324
2325        with ensure_clean_store(self.path) as store:                 2325        with ensure_clean_store(self.path) as store:
2326                                                                     2326
2327            with catch_warnings(record=True):                        2327            with catch_warnings(record=True):
2328                wp = tm.makePanel(30)                                2328                wp = tm.makePanel(30)
2329                                                                     2329
2330                # start                                              2330                # start
2331                _maybe_remove(store, 'wp1')                          2331                _maybe_remove(store, 'wp1')
2332                store.put('wp1', wp, format='t')                     2332                store.put('wp1', wp, format='t')
2333                n = store.remove('wp1', start=32)                    2333                n = store.remove('wp1', start=32)
2334                assert n == 120 - 32                                 2334                assert n == 120 - 32
2335                result = store.select('wp1')                         2335                result = store.select('wp1')
2336                expected = wp.reindex(major_axis=wp.major_axis[:32 /2336                expected = wp.reindex(major_axis=wp.major_axis[:32 /
                                                                   / 4])                                                                   / 4])
2337                assert_panel_equal(result, expected)                 2337                assert_panel_equal(result, expected)
2338                                                                     2338
2339                _maybe_remove(store, 'wp2')                          2339                _maybe_remove(store, 'wp2')
2340                store.put('wp2', wp, format='t')                     2340                store.put('wp2', wp, format='t')
2341                n = store.remove('wp2', start=-32)                   2341                n = store.remove('wp2', start=-32)
2342                assert n == 32                                       2342                assert n == 32
2343                result = store.select('wp2')                         2343                result = store.select('wp2')
2344                expected = wp.reindex(major_axis=wp.major_axis[:-32 2344                expected = wp.reindex(major_axis=wp.major_axis[:-32 
                                                                  // 4])                                                                  // 4])
2345                assert_panel_equal(result, expected)                 2345                assert_panel_equal(result, expected)
2346                                                                     2346
2347                # stop                                               2347                # stop
2348                _maybe_remove(store, 'wp3')                          2348                _maybe_remove(store, 'wp3')
2349                store.put('wp3', wp, format='t')                     2349                store.put('wp3', wp, format='t')
2350                n = store.remove('wp3', stop=32)                     2350                n = store.remove('wp3', stop=32)
2351                assert n == 32                                       2351                assert n == 32
2352                result = store.select('wp3')                         2352                result = store.select('wp3')
2353                expected = wp.reindex(major_axis=wp.major_axis[32 //2353                expected = wp.reindex(major_axis=wp.major_axis[32 //
                                                                    4:])                                                                    4:])
2354                assert_panel_equal(result, expected)                 2354                assert_panel_equal(result, expected)
2355                                                                     2355
2356                _maybe_remove(store, 'wp4')                          2356                _maybe_remove(store, 'wp4')
2357                store.put('wp4', wp, format='t')                     2357                store.put('wp4', wp, format='t')
2358                n = store.remove('wp4', stop=-32)                    2358                n = store.remove('wp4', stop=-32)
2359                assert n == 120 - 32                                 2359                assert n == 120 - 32
2360                result = store.select('wp4')                         2360                result = store.select('wp4')
2361                expected = wp.reindex(major_axis=wp.major_axis[-32 /2361                expected = wp.reindex(major_axis=wp.major_axis[-32 /
                                                                  / 4:])                                                                  / 4:])
2362                assert_panel_equal(result, expected)                 2362                assert_panel_equal(result, expected)
2363                                                                     2363
2364                # start n stop                                       2364                # start n stop
2365                _maybe_remove(store, 'wp5')                          2365                _maybe_remove(store, 'wp5')
2366                store.put('wp5', wp, format='t')                     2366                store.put('wp5', wp, format='t')
2367                n = store.remove('wp5', start=16, stop=-16)          2367                n = store.remove('wp5', start=16, stop=-16)
2368                assert n == 120 - 32                                 2368                assert n == 120 - 32
2369                result = store.select('wp5')                         2369                result = store.select('wp5')
2370                expected = wp.reindex(                               2370                expected = wp.reindex(
2371                    major_axis=(wp.major_axis[:16 // 4]              2371                    major_axis=(wp.major_axis[:16 // 4]
2372                                .union(wp.major_axis[-16 // 4:])))   2372                                .union(wp.major_axis[-16 // 4:])))
2373                assert_panel_equal(result, expected)                 2373                assert_panel_equal(result, expected)
2374                                                                     2374
2375                _maybe_remove(store, 'wp6')                          2375                _maybe_remove(store, 'wp6')
2376                store.put('wp6', wp, format='t')                     2376                store.put('wp6', wp, format='t')
2377                n = store.remove('wp6', start=16, stop=16)           2377                n = store.remove('wp6', start=16, stop=16)
2378                assert n == 0                                        2378                assert n == 0
2379                result = store.select('wp6')                         2379                result = store.select('wp6')
2380                expected = wp.reindex(major_axis=wp.major_axis)      2380                expected = wp.reindex(major_axis=wp.major_axis)
2381                assert_panel_equal(result, expected)                 2381                assert_panel_equal(result, expected)
2382                                                                     2382
2383                # with where                                         2383                # with where
2384                _maybe_remove(store, 'wp7')                          2384                _maybe_remove(store, 'wp7')
2385                                                                     2385
2386                # TODO: unused?                                      2386                # TODO: unused?
2387                date = wp.major_axis.take(np.arange(0, 30, 3))  # no2387                date = wp.major_axis.take(np.arange(0, 30, 3))  # no
                                                                      qa                                                                      qa
2388                                                                     2388
2389                crit = 'major_axis=date'                             2389                crit = 'major_axis=date'
2390                store.put('wp7', wp, format='t')                     2390                store.put('wp7', wp, format='t')
2391                n = store.remove('wp7', where=[crit], stop=80)       2391                n = store.remove('wp7', where=[crit], stop=80)
2392                assert n == 28                                       2392                assert n == 28
2393                result = store.select('wp7')                         2393                result = store.select('wp7')
2394                expected = wp.reindex(major_axis=wp.major_axis.diffe2394                expected = wp.reindex(major_axis=wp.major_axis.diffe
                                                                  rence(                                                                  rence(
2395                    wp.major_axis[np.arange(0, 20, 3)]))             2395                    wp.major_axis[np.arange(0, 20, 3)]))
2396                assert_panel_equal(result, expected)                 2396                assert_panel_equal(result, expected)
2397                                                                     2397
2398    def test_remove_crit(self):                                      2398    def test_remove_crit(self):
2399                                                                     2399
2400        with ensure_clean_store(self.path) as store:                 2400        with ensure_clean_store(self.path) as store:
2401                                                                     2401
2402            with catch_warnings(record=True):                        2402            with catch_warnings(record=True):
2403                wp = tm.makePanel(30)                                2403                wp = tm.makePanel(30)
2404                                                                     2404
2405                # group row removal                                  2405                # group row removal
2406                _maybe_remove(store, 'wp3')                          2406                _maybe_remove(store, 'wp3')
2407                date4 = wp.major_axis.take([0, 1, 2, 4, 5, 6, 8, 9, 2407                date4 = wp.major_axis.take([0, 1, 2, 4, 5, 6, 8, 9, 
                                                                    10])                                                                    10])
2408                crit4 = 'major_axis=date4'                           2408                crit4 = 'major_axis=date4'
2409                store.put('wp3', wp, format='t')                     2409                store.put('wp3', wp, format='t')
2410                n = store.remove('wp3', where=[crit4])               2410                n = store.remove('wp3', where=[crit4])
2411                assert n == 36                                       2411                assert n == 36
2412                                                                     2412
2413                result = store.select('wp3')                         2413                result = store.select('wp3')
2414                expected = wp.reindex(                               2414                expected = wp.reindex(
2415                    major_axis=wp.major_axis.difference(date4))      2415                    major_axis=wp.major_axis.difference(date4))
2416                assert_panel_equal(result, expected)                 2416                assert_panel_equal(result, expected)
2417                                                                     2417
2418                # upper half                                         2418                # upper half
2419                _maybe_remove(store, 'wp')                           2419                _maybe_remove(store, 'wp')
2420                store.put('wp', wp, format='table')                  2420                store.put('wp', wp, format='table')
2421                date = wp.major_axis[len(wp.major_axis) // 2]        2421                date = wp.major_axis[len(wp.major_axis) // 2]
2422                                                                     2422
2423                crit1 = 'major_axis>date'                            2423                crit1 = 'major_axis>date'
2424                crit2 = "minor_axis=['A', 'D']"                      2424                crit2 = "minor_axis=['A', 'D']"
2425                n = store.remove('wp', where=[crit1])                2425                n = store.remove('wp', where=[crit1])
2426                assert n == 56                                       2426                assert n == 56
2427                                                                     2427
2428                n = store.remove('wp', where=[crit2])                2428                n = store.remove('wp', where=[crit2])
2429                assert n == 32                                       2429                assert n == 32
2430                                                                     2430
2431                result = store['wp']                                 2431                result = store['wp']
2432                expected = wp.truncate(after=date).reindex(minor=['B2432                expected = wp.truncate(after=date).reindex(minor=['B
                                                                ', 'C'])                                                                ', 'C'])
2433                assert_panel_equal(result, expected)                 2433                assert_panel_equal(result, expected)
2434                                                                     2434
2435                # individual row elements                            2435                # individual row elements
2436                _maybe_remove(store, 'wp2')                          2436                _maybe_remove(store, 'wp2')
2437                store.put('wp2', wp, format='table')                 2437                store.put('wp2', wp, format='table')
2438                                                                     2438
2439                date1 = wp.major_axis[1:3]                           2439                date1 = wp.major_axis[1:3]
2440                crit1 = 'major_axis=date1'                           2440                crit1 = 'major_axis=date1'
2441                store.remove('wp2', where=[crit1])                   2441                store.remove('wp2', where=[crit1])
2442                result = store.select('wp2')                         2442                result = store.select('wp2')
2443                expected = wp.reindex(                               2443                expected = wp.reindex(
2444                    major_axis=wp.major_axis.difference(date1))      2444                    major_axis=wp.major_axis.difference(date1))
2445                assert_panel_equal(result, expected)                 2445                assert_panel_equal(result, expected)
2446                                                                     2446
2447                date2 = wp.major_axis[5]                             2447                date2 = wp.major_axis[5]
2448                crit2 = 'major_axis=date2'                           2448                crit2 = 'major_axis=date2'
2449                store.remove('wp2', where=[crit2])                   2449                store.remove('wp2', where=[crit2])
2450                result = store['wp2']                                2450                result = store['wp2']
2451                expected = wp.reindex(                               2451                expected = wp.reindex(
2452                    major_axis=(wp.major_axis                        2452                    major_axis=(wp.major_axis
2453                                .difference(date1)                   2453                                .difference(date1)
2454                                .difference(Index([date2]))          2454                                .difference(Index([date2]))
2455                                ))                                   2455                                ))
2456                assert_panel_equal(result, expected)                 2456                assert_panel_equal(result, expected)
2457                                                                     2457
2458                date3 = [wp.major_axis[7], wp.major_axis[9]]         2458                date3 = [wp.major_axis[7], wp.major_axis[9]]
2459                crit3 = 'major_axis=date3'                           2459                crit3 = 'major_axis=date3'
2460                store.remove('wp2', where=[crit3])                   2460                store.remove('wp2', where=[crit3])
2461                result = store['wp2']                                2461                result = store['wp2']
2462                expected = wp.reindex(major_axis=wp.major_axis       2462                expected = wp.reindex(major_axis=wp.major_axis
2463                                      .difference(date1)             2463                                      .difference(date1)
2464                                      .difference(Index([date2]))    2464                                      .difference(Index([date2]))
2465                                      .difference(Index(date3)))     2465                                      .difference(Index(date3)))
2466                assert_panel_equal(result, expected)                 2466                assert_panel_equal(result, expected)
2467                                                                     2467
2468                # corners                                            2468                # corners
2469                _maybe_remove(store, 'wp4')                          2469                _maybe_remove(store, 'wp4')
2470                store.put('wp4', wp, format='table')                 2470                store.put('wp4', wp, format='table')
2471                n = store.remove(                                    2471                n = store.remove(
2472                    'wp4', where="major_axis>wp.major_axis[-1]")     2472                    'wp4', where="major_axis>wp.major_axis[-1]")
2473                result = store.select('wp4')                         2473                result = store.select('wp4')
2474                assert_panel_equal(result, wp)                       2474                assert_panel_equal(result, wp)
2475                                                                     2475
2476    def test_invalid_terms(self):                                    2476    def test_invalid_terms(self):
2477                                                                     2477
2478        with ensure_clean_store(self.path) as store:                 2478        with ensure_clean_store(self.path) as store:
2479                                                                     2479
2480            with catch_warnings(record=True):                        2480            with catch_warnings(record=True):
2481                                                                     2481
2482                df = tm.makeTimeDataFrame()                          2482                df = tm.makeTimeDataFrame()
2483                df['string'] = 'foo'                                 2483                df['string'] = 'foo'
2484                df.loc[0:4, 'string'] = 'bar'                        2484                df.loc[0:4, 'string'] = 'bar'
2485                wp = tm.makePanel()                                  2485                wp = tm.makePanel()
2486                                                                     2486
2487                p4d = tm.makePanel4D()                               2487                p4d = tm.makePanel4D()
2488                store.put('df', df, format='table')                  2488                store.put('df', df, format='table')
2489                store.put('wp', wp, format='table')                  2489                store.put('wp', wp, format='table')
2490                store.put('p4d', p4d, format='table')                2490                store.put('p4d', p4d, format='table')
2491                                                                     2491
2492                # some invalid terms                                 2492                # some invalid terms
2493                pytest.raises(ValueError, store.select,              2493                pytest.raises(ValueError, store.select,
2494                              'wp', "minor=['A', 'B']")              2494                              'wp', "minor=['A', 'B']")
2495                pytest.raises(ValueError, store.select,              2495                pytest.raises(ValueError, store.select,
2496                              'wp', ["index=['20121114']"])          2496                              'wp', ["index=['20121114']"])
2497                pytest.raises(ValueError, store.select, 'wp', [      2497                pytest.raises(ValueError, store.select, 'wp', [
2498                    "index=['20121114', '20121114']"])               2498                    "index=['20121114', '20121114']"])
2499                pytest.raises(TypeError, Term)                       2499                pytest.raises(TypeError, Term)
2500                                                                     2500
2501                # more invalid                                       2501                # more invalid
2502                pytest.raises(                                       2502                pytest.raises(
2503                    ValueError, store.select, 'df', 'df.index[3]')   2503                    ValueError, store.select, 'df', 'df.index[3]')
2504                pytest.raises(SyntaxError, store.select, 'df', 'inde2504                pytest.raises(SyntaxError, store.select, 'df', 'inde
                                                                    x>')                                                                    x>')
2505                pytest.raises(                                       2505                pytest.raises(
2506                    ValueError, store.select, 'wp',                  2506                    ValueError, store.select, 'wp',
2507                    "major_axis<'20000108' & minor_axis['A', 'B']")  2507                    "major_axis<'20000108' & minor_axis['A', 'B']")
2508                                                                     2508
2509        # from the docs                                              2509        # from the docs
2510        with ensure_clean_path(self.path) as path:                   2510        with ensure_clean_path(self.path) as path:
2511            dfq = DataFrame(np.random.randn(10, 4), columns=list(    2511            dfq = DataFrame(np.random.randn(10, 4), columns=list(
2512                'ABCD'), index=date_range('20130101', periods=10))   2512                'ABCD'), index=date_range('20130101', periods=10))
2513            dfq.to_hdf(path, 'dfq', format='table', data_columns=Tru2513            dfq.to_hdf(path, 'dfq', format='table', data_columns=Tru
                                                                      e)                                                                      e)
2514                                                                     2514
2515            # check ok                                               2515            # check ok
2516            read_hdf(path, 'dfq',                                    2516            read_hdf(path, 'dfq',
2517                     where="index>Timestamp('20130104') & columns=['2517                     where="index>Timestamp('20130104') & columns=['
                                                              A', 'B']")                                                              A', 'B']")
2518            read_hdf(path, 'dfq', where="A>0 or C>0")                2518            read_hdf(path, 'dfq', where="A>0 or C>0")
2519                                                                     2519
2520        # catch the invalid reference                                2520        # catch the invalid reference
2521        with ensure_clean_path(self.path) as path:                   2521        with ensure_clean_path(self.path) as path:
2522            dfq = DataFrame(np.random.randn(10, 4), columns=list(    2522            dfq = DataFrame(np.random.randn(10, 4), columns=list(
2523                'ABCD'), index=date_range('20130101', periods=10))   2523                'ABCD'), index=date_range('20130101', periods=10))
2524            dfq.to_hdf(path, 'dfq', format='table')                  2524            dfq.to_hdf(path, 'dfq', format='table')
2525                                                                     2525
2526            pytest.raises(ValueError, read_hdf, path,                2526            pytest.raises(ValueError, read_hdf, path,
2527                          'dfq', where="A>0 or C>0")                 2527                          'dfq', where="A>0 or C>0")
2528                                                                     2528
2529    def test_terms(self):                                            2529    def test_terms(self):
2530                                                                     2530
2531        with ensure_clean_store(self.path) as store:                 2531        with ensure_clean_store(self.path) as store:
2532                                                                     2532
2533            with catch_warnings(record=True):                        2533            with catch_warnings(record=True):
2534                                                                     2534
2535                wp = tm.makePanel()                                  2535                wp = tm.makePanel()
2536                wpneg = Panel.fromDict({-1: tm.makeDataFrame(),      2536                wpneg = Panel.fromDict({-1: tm.makeDataFrame(),
2537                                        0: tm.makeDataFrame(),       2537                                        0: tm.makeDataFrame(),
2538                                        1: tm.makeDataFrame()})      2538                                        1: tm.makeDataFrame()})
2539                p4d = tm.makePanel4D()                               2539                p4d = tm.makePanel4D()
2540                store.put('p4d', p4d, format='table')                2540                store.put('p4d', p4d, format='table')
2541                store.put('wp', wp, format='table')                  2541                store.put('wp', wp, format='table')
2542                store.put('wpneg', wpneg, format='table')            2542                store.put('wpneg', wpneg, format='table')
2543                                                                     2543
2544                # panel                                              2544                # panel
2545                result = store.select(                               2545                result = store.select(
2546                    'wp',                                            2546                    'wp',
2547                    "major_axis<'20000108' and minor_axis=['A', 'B']2547                    "major_axis<'20000108' and minor_axis=['A', 'B']
                                                                      ")                                                                      ")
2548                expected = wp.truncate(                              2548                expected = wp.truncate(
2549                    after='20000108').reindex(minor=['A', 'B'])      2549                    after='20000108').reindex(minor=['A', 'B'])
2550                assert_panel_equal(result, expected)                 2550                assert_panel_equal(result, expected)
2551                                                                     2551
2552                # with deprecation                                   2552                # with deprecation
2553                result = store.select(                               2553                result = store.select(
2554                    'wp', where=("major_axis<'20000108' "            2554                    'wp', where=("major_axis<'20000108' "
2555                                 "and minor_axis=['A', 'B']"))       2555                                 "and minor_axis=['A', 'B']"))
2556                expected = wp.truncate(                              2556                expected = wp.truncate(
2557                    after='20000108').reindex(minor=['A', 'B'])      2557                    after='20000108').reindex(minor=['A', 'B'])
2558                tm.assert_panel_equal(result, expected)              2558                tm.assert_panel_equal(result, expected)
2559                                                                     2559
2560            # p4d                                                    2560            # p4d
2561            with catch_warnings(record=True):                        2561            with catch_warnings(record=True):
2562                                                                     2562
2563                result = store.select('p4d',                         2563                result = store.select('p4d',
2564                                      ("major_axis<'20000108' and "  2564                                      ("major_axis<'20000108' and "
2565                                       "minor_axis=['A', 'B'] and "  2565                                       "minor_axis=['A', 'B'] and "
2566                                       "items=['ItemA', 'ItemB']"))  2566                                       "items=['ItemA', 'ItemB']"))
2567                expected = p4d.truncate(after='20000108').reindex(   2567                expected = p4d.truncate(after='20000108').reindex(
2568                    minor=['A', 'B'], items=['ItemA', 'ItemB'])      2568                    minor=['A', 'B'], items=['ItemA', 'ItemB'])
2569                assert_panel4d_equal(result, expected)               2569                assert_panel4d_equal(result, expected)
2570                                                                     2570
2571            with catch_warnings(record=True):                        2571            with catch_warnings(record=True):
2572                                                                     2572
2573                # valid terms                                        2573                # valid terms
2574                terms = [('major_axis=20121114'),                    2574                terms = [('major_axis=20121114'),
2575                         ('major_axis>20121114'),                    2575                         ('major_axis>20121114'),
2576                         (("major_axis=['20121114', '20121114']"),), 2576                         (("major_axis=['20121114', '20121114']"),),
2577                         ('major_axis=datetime.datetime(2012, 11, 142577                         ('major_axis=datetime.datetime(2012, 11, 14
                                                                    )'),                                                                    )'),
2578                         'major_axis> 20121114',                     2578                         'major_axis> 20121114',
2579                         'major_axis >20121114',                     2579                         'major_axis >20121114',
2580                         'major_axis > 20121114',                    2580                         'major_axis > 20121114',
2581                         (("minor_axis=['A', 'B']"),),               2581                         (("minor_axis=['A', 'B']"),),
2582                         (("minor_axis=['A', 'B']"),),               2582                         (("minor_axis=['A', 'B']"),),
2583                         ((("minor_axis==['A', 'B']"),),),           2583                         ((("minor_axis==['A', 'B']"),),),
2584                         (("items=['ItemA', 'ItemB']"),),            2584                         (("items=['ItemA', 'ItemB']"),),
2585                         ('items=ItemA'),                            2585                         ('items=ItemA'),
2586                         ]                                           2586                         ]
2587                                                                     2587
2588                for t in terms:                                      2588                for t in terms:
2589                    store.select('wp', t)                            2589                    store.select('wp', t)
2590                    store.select('p4d', t)                           2590                    store.select('p4d', t)
2591                                                                     2591
2592                # valid for p4d only                                 2592                # valid for p4d only
2593                terms = ["labels=['l1', 'l2']"]                      2593                terms = ["labels=['l1', 'l2']"]
2594                for t in terms:                                      2594                for t in terms:
2595                    store.select('p4d', t)                           2595                    store.select('p4d', t)
2596                                                                     2596
2597                with tm.assert_raises_regex(                         2597                with tm.assert_raises_regex(
2598                        TypeError, 'Only named functions are support2598                        TypeError, 'Only named functions are support
                                                                   ed'):                                                                   ed'):
2599                    store.select(                                    2599                    store.select(
2600                        'wp',                                        2600                        'wp',
2601                        'major_axis == (lambda x: x)("20130101")')   2601                        'major_axis == (lambda x: x)("20130101")')
2602                                                                     2602
2603            with catch_warnings(record=True):                        2603            with catch_warnings(record=True):
2604                # check USub node parsing                            2604                # check USub node parsing
2605                res = store.select('wpneg', 'items == -1')           2605                res = store.select('wpneg', 'items == -1')
2606                expected = Panel({-1: wpneg[-1]})                    2606                expected = Panel({-1: wpneg[-1]})
2607                tm.assert_panel_equal(res, expected)                 2607                tm.assert_panel_equal(res, expected)
2608                                                                     2608
2609                with tm.assert_raises_regex(NotImplementedError,     2609                with tm.assert_raises_regex(NotImplementedError,
2610                                            'Unary addition '        2610                                            'Unary addition '
2611                                            'not supported'):        2611                                            'not supported'):
2612                    store.select('wpneg', 'items == +1')             2612                    store.select('wpneg', 'items == +1')
2613                                                                     2613
2614    def test_term_compat(self):                                      2614    def test_term_compat(self):
2615        with ensure_clean_store(self.path) as store:                 2615        with ensure_clean_store(self.path) as store:
2616                                                                     2616
2617            with catch_warnings(record=True):                        2617            with catch_warnings(record=True):
2618                wp = Panel(np.random.randn(2, 5, 4), items=['Item1',2618                wp = Panel(np.random.randn(2, 5, 4), items=['Item1',
                                                               'Item2'],                                                               'Item2'],
2619                           major_axis=date_range('1/1/2000', periods2619                           major_axis=date_range('1/1/2000', periods
                                                                    =5),                                                                    =5),
2620                           minor_axis=['A', 'B', 'C', 'D'])          2620                           minor_axis=['A', 'B', 'C', 'D'])
2621                store.append('wp', wp)                               2621                store.append('wp', wp)
2622                                                                     2622
2623                result = store.select(                               2623                result = store.select(
2624                    'wp', where=("major_axis>20000102 "              2624                    'wp', where=("major_axis>20000102 "
2625                                 "and minor_axis=['A', 'B']"))       2625                                 "and minor_axis=['A', 'B']"))
2626                expected = wp.loc[:, wp.major_axis >                 2626                expected = wp.loc[:, wp.major_axis >
2627                                  Timestamp('20000102'), ['A', 'B']] 2627                                  Timestamp('20000102'), ['A', 'B']]
2628                assert_panel_equal(result, expected)                 2628                assert_panel_equal(result, expected)
2629                                                                     2629
2630                store.remove('wp', 'major_axis>20000103')            2630                store.remove('wp', 'major_axis>20000103')
2631                result = store.select('wp')                          2631                result = store.select('wp')
2632                expected = wp.loc[:, wp.major_axis <= Timestamp('2002632                expected = wp.loc[:, wp.major_axis <= Timestamp('200
                                                             00103'), :]                                                             00103'), :]
2633                assert_panel_equal(result, expected)                 2633                assert_panel_equal(result, expected)
2634                                                                     2634
2635        with ensure_clean_store(self.path) as store:                 2635        with ensure_clean_store(self.path) as store:
2636                                                                     2636
2637            with catch_warnings(record=True):                        2637            with catch_warnings(record=True):
2638                wp = Panel(np.random.randn(2, 5, 4),                 2638                wp = Panel(np.random.randn(2, 5, 4),
2639                           items=['Item1', 'Item2'],                 2639                           items=['Item1', 'Item2'],
2640                           major_axis=date_range('1/1/2000', periods2640                           major_axis=date_range('1/1/2000', periods
                                                                    =5),                                                                    =5),
2641                           minor_axis=['A', 'B', 'C', 'D'])          2641                           minor_axis=['A', 'B', 'C', 'D'])
2642                store.append('wp', wp)                               2642                store.append('wp', wp)
2643                                                                     2643
2644                # stringified datetimes                              2644                # stringified datetimes
2645                result = store.select(                               2645                result = store.select(
2646                    'wp', 'major_axis>datetime.datetime(2000, 1, 2)')2646                    'wp', 'major_axis>datetime.datetime(2000, 1, 2)')
2647                expected = wp.loc[:, wp.major_axis > Timestamp('20002647                expected = wp.loc[:, wp.major_axis > Timestamp('2000
                                                                 0102')]                                                                 0102')]
2648                assert_panel_equal(result, expected)                 2648                assert_panel_equal(result, expected)
2649                                                                     2649
2650                result = store.select(                               2650                result = store.select(
2651                    'wp', 'major_axis>datetime.datetime(2000, 1, 2)')2651                    'wp', 'major_axis>datetime.datetime(2000, 1, 2)')
2652                expected = wp.loc[:, wp.major_axis > Timestamp('20002652                expected = wp.loc[:, wp.major_axis > Timestamp('2000
                                                                 0102')]                                                                 0102')]
2653                assert_panel_equal(result, expected)                 2653                assert_panel_equal(result, expected)
2654                                                                     2654
2655                result = store.select(                               2655                result = store.select(
2656                    'wp',                                            2656                    'wp',
2657                    "major_axis=[datetime.datetime(2000, 1, 2, 0, 0)2657                    "major_axis=[datetime.datetime(2000, 1, 2, 0, 0)
                                                                     , "                                                                     , "
2658                    "datetime.datetime(2000, 1, 3, 0, 0)]")          2658                    "datetime.datetime(2000, 1, 3, 0, 0)]")
2659                expected = wp.loc[:, [Timestamp('20000102'),         2659                expected = wp.loc[:, [Timestamp('20000102'),
2660                                      Timestamp('20000103')]]        2660                                      Timestamp('20000103')]]
2661                assert_panel_equal(result, expected)                 2661                assert_panel_equal(result, expected)
2662                                                                     2662
2663                result = store.select(                               2663                result = store.select(
2664                    'wp', "minor_axis=['A', 'B']")                   2664                    'wp', "minor_axis=['A', 'B']")
2665                expected = wp.loc[:, :, ['A', 'B']]                  2665                expected = wp.loc[:, :, ['A', 'B']]
2666                assert_panel_equal(result, expected)                 2666                assert_panel_equal(result, expected)
2667                                                                     2667
2668    def test_same_name_scoping(self):                                2668    def test_same_name_scoping(self):
2669                                                                     2669
2670        with ensure_clean_store(self.path) as store:                 2670        with ensure_clean_store(self.path) as store:
2671                                                                     2671
2672            import pandas as pd                                      2672            import pandas as pd
2673            df = DataFrame(np.random.randn(20, 2),                   2673            df = DataFrame(np.random.randn(20, 2),
2674                           index=pd.date_range('20130101', periods=22674                           index=pd.date_range('20130101', periods=2
                                                                     0))                                                                     0))
2675            store.put('df', df, format='table')                      2675            store.put('df', df, format='table')
2676            expected = df[df.index > pd.Timestamp('20130105')]       2676            expected = df[df.index > pd.Timestamp('20130105')]
2677                                                                     2677
2678            import datetime  # noqa                                  2678            import datetime  # noqa
2679            result = store.select('df', 'index>datetime.datetime(2012679            result = store.select('df', 'index>datetime.datetime(201
                                                                3,1,5)')                                                                3,1,5)')
2680            assert_frame_equal(result, expected)                     2680            assert_frame_equal(result, expected)
2681                                                                     2681
2682            from datetime import datetime  # noqa                    2682            from datetime import datetime  # noqa
2683                                                                     2683
2684            # technically an error, but allow it                     2684            # technically an error, but allow it
2685            result = store.select('df', 'index>datetime.datetime(2012685            result = store.select('df', 'index>datetime.datetime(201
                                                                3,1,5)')                                                                3,1,5)')
2686            assert_frame_equal(result, expected)                     2686            assert_frame_equal(result, expected)
2687                                                                     2687
2688            result = store.select('df', 'index>datetime(2013,1,5)')  2688            result = store.select('df', 'index>datetime(2013,1,5)')
2689            assert_frame_equal(result, expected)                     2689            assert_frame_equal(result, expected)
2690                                                                     2690
2691    def test_series(self):                                           2691    def test_series(self):
2692                                                                     2692
2693        s = tm.makeStringSeries()                                    2693        s = tm.makeStringSeries()
2694        self._check_roundtrip(s, tm.assert_series_equal)             2694        self._check_roundtrip(s, tm.assert_series_equal)
2695                                                                     2695
2696        ts = tm.makeTimeSeries()                                     2696        ts = tm.makeTimeSeries()
2697        self._check_roundtrip(ts, tm.assert_series_equal)            2697        self._check_roundtrip(ts, tm.assert_series_equal)
2698                                                                     2698
2699        ts2 = Series(ts.index, Index(ts.index, dtype=object))        2699        ts2 = Series(ts.index, Index(ts.index, dtype=object))
2700        self._check_roundtrip(ts2, tm.assert_series_equal)           2700        self._check_roundtrip(ts2, tm.assert_series_equal)
2701                                                                     2701
2702        ts3 = Series(ts.values, Index(np.asarray(ts.index, dtype=obj2702        ts3 = Series(ts.values, Index(np.asarray(ts.index, dtype=obj
                                                                   ect),                                                                   ect),
2703                                      dtype=object))                 2703                                      dtype=object))
2704        self._check_roundtrip(ts3, tm.assert_series_equal,           2704        self._check_roundtrip(ts3, tm.assert_series_equal,
2705                              check_index_type=False)                2705                              check_index_type=False)
2706                                                                     2706
2707    def test_sparse_series(self):                                    2707    def test_sparse_series(self):
2708                                                                     2708
2709        s = tm.makeStringSeries()                                    2709        s = tm.makeStringSeries()
2710        s.iloc[3:5] = np.nan                                         2710        s.iloc[3:5] = np.nan
2711        ss = s.to_sparse()                                           2711        ss = s.to_sparse()
2712        self._check_roundtrip(ss, tm.assert_series_equal,            2712        self._check_roundtrip(ss, tm.assert_series_equal,
2713                              check_series_type=True)                2713                              check_series_type=True)
2714                                                                     2714
2715        ss2 = s.to_sparse(kind='integer')                            2715        ss2 = s.to_sparse(kind='integer')
2716        self._check_roundtrip(ss2, tm.assert_series_equal,           2716        self._check_roundtrip(ss2, tm.assert_series_equal,
2717                              check_series_type=True)                2717                              check_series_type=True)
2718                                                                     2718
2719        ss3 = s.to_sparse(fill_value=0)                              2719        ss3 = s.to_sparse(fill_value=0)
2720        self._check_roundtrip(ss3, tm.assert_series_equal,           2720        self._check_roundtrip(ss3, tm.assert_series_equal,
2721                              check_series_type=True)                2721                              check_series_type=True)
2722                                                                     2722
2723    def test_sparse_frame(self):                                     2723    def test_sparse_frame(self):
2724                                                                     2724
2725        s = tm.makeDataFrame()                                       2725        s = tm.makeDataFrame()
2726        s.iloc[3:5, 1:3] = np.nan                                    2726        s.iloc[3:5, 1:3] = np.nan
2727        s.iloc[8:10, -2] = np.nan                                    2727        s.iloc[8:10, -2] = np.nan
2728        ss = s.to_sparse()                                           2728        ss = s.to_sparse()
2729                                                                     2729
2730        self._check_double_roundtrip(ss, tm.assert_frame_equal,      2730        self._check_double_roundtrip(ss, tm.assert_frame_equal,
2731                                     check_frame_type=True)          2731                                     check_frame_type=True)
2732                                                                     2732
2733        ss2 = s.to_sparse(kind='integer')                            2733        ss2 = s.to_sparse(kind='integer')
2734        self._check_double_roundtrip(ss2, tm.assert_frame_equal,     2734        self._check_double_roundtrip(ss2, tm.assert_frame_equal,
2735                                     check_frame_type=True)          2735                                     check_frame_type=True)
2736                                                                     2736
2737        ss3 = s.to_sparse(fill_value=0)                              2737        ss3 = s.to_sparse(fill_value=0)
2738        self._check_double_roundtrip(ss3, tm.assert_frame_equal,     2738        self._check_double_roundtrip(ss3, tm.assert_frame_equal,
2739                                     check_frame_type=True)          2739                                     check_frame_type=True)
2740                                                                     2740
2741    def test_float_index(self):                                      2741    def test_float_index(self):
2742                                                                     2742
2743        # GH #454                                                    2743        # GH #454
2744        index = np.random.randn(10)                                  2744        index = np.random.randn(10)
2745        s = Series(np.random.randn(10), index=index)                 2745        s = Series(np.random.randn(10), index=index)
2746        self._check_roundtrip(s, tm.assert_series_equal)             2746        self._check_roundtrip(s, tm.assert_series_equal)
2747                                                                     2747
2748    def test_tuple_index(self):                                      2748    def test_tuple_index(self):
2749                                                                     2749
2750        # GH #492                                                    2750        # GH #492
2751        col = np.arange(10)                                          2751        col = np.arange(10)
2752        idx = [(0., 1.), (2., 3.), (4., 5.)]                         2752        idx = [(0., 1.), (2., 3.), (4., 5.)]
2753        data = np.random.randn(30).reshape((3, 10))                  2753        data = np.random.randn(30).reshape((3, 10))
2754        DF = DataFrame(data, index=idx, columns=col)                 2754        DF = DataFrame(data, index=idx, columns=col)
2755                                                                     2755
2756        with catch_warnings(record=True):                            2756        with catch_warnings(record=True):
2757            self._check_roundtrip(DF, tm.assert_frame_equal)         2757            self._check_roundtrip(DF, tm.assert_frame_equal)
2758                                                                     2758
2759    def test_index_types(self):                                      2759    def test_index_types(self):
2760                                                                     2760
2761        with catch_warnings(record=True):                            2761        with catch_warnings(record=True):
2762            values = np.random.randn(2)                              2762            values = np.random.randn(2)
2763                                                                     2763
2764            func = lambda l, r: tm.assert_series_equal(l, r,         2764            func = lambda l, r: tm.assert_series_equal(l, r,
2765                                                       check_dtype=T2765                                                       check_dtype=T
                                                                    rue,                                                                    rue,
2766                                                       check_index_t2766                                                       check_index_t
                                                               ype=True,                                                               ype=True,
2767                                                       check_series_2767                                                       check_series_
                                                              type=True)                                                              type=True)
2768                                                                     2768
2769        with catch_warnings(record=True):                            2769        with catch_warnings(record=True):
2770            ser = Series(values, [0, 'y'])                           2770            ser = Series(values, [0, 'y'])
2771            self._check_roundtrip(ser, func)                         2771            self._check_roundtrip(ser, func)
2772                                                                     2772
2773        with catch_warnings(record=True):                            2773        with catch_warnings(record=True):
2774            ser = Series(values, [datetime.datetime.today(), 0])     2774            ser = Series(values, [datetime.datetime.today(), 0])
2775            self._check_roundtrip(ser, func)                         2775            self._check_roundtrip(ser, func)
2776                                                                     2776
2777        with catch_warnings(record=True):                            2777        with catch_warnings(record=True):
2778            ser = Series(values, ['y', 0])                           2778            ser = Series(values, ['y', 0])
2779            self._check_roundtrip(ser, func)                         2779            self._check_roundtrip(ser, func)
2780                                                                     2780
2781        with catch_warnings(record=True):                            2781        with catch_warnings(record=True):
2782            ser = Series(values, [datetime.date.today(), 'a'])       2782            ser = Series(values, [datetime.date.today(), 'a'])
2783            self._check_roundtrip(ser, func)                         2783            self._check_roundtrip(ser, func)
2784                                                                     2784
2785        with catch_warnings(record=True):                            2785        with catch_warnings(record=True):
2786                                                                     2786
2787            ser = Series(values, [0, 'y'])                           2787            ser = Series(values, [0, 'y'])
2788            self._check_roundtrip(ser, func)                         2788            self._check_roundtrip(ser, func)
2789                                                                     2789
2790            ser = Series(values, [datetime.datetime.today(), 0])     2790            ser = Series(values, [datetime.datetime.today(), 0])
2791            self._check_roundtrip(ser, func)                         2791            self._check_roundtrip(ser, func)
2792                                                                     2792
2793            ser = Series(values, ['y', 0])                           2793            ser = Series(values, ['y', 0])
2794            self._check_roundtrip(ser, func)                         2794            self._check_roundtrip(ser, func)
2795                                                                     2795
2796            ser = Series(values, [datetime.date.today(), 'a'])       2796            ser = Series(values, [datetime.date.today(), 'a'])
2797            self._check_roundtrip(ser, func)                         2797            self._check_roundtrip(ser, func)
2798                                                                     2798
2799            ser = Series(values, [1.23, 'b'])                        2799            ser = Series(values, [1.23, 'b'])
2800            self._check_roundtrip(ser, func)                         2800            self._check_roundtrip(ser, func)
2801                                                                     2801
2802            ser = Series(values, [1, 1.53])                          2802            ser = Series(values, [1, 1.53])
2803            self._check_roundtrip(ser, func)                         2803            self._check_roundtrip(ser, func)
2804                                                                     2804
2805            ser = Series(values, [1, 5])                             2805            ser = Series(values, [1, 5])
2806            self._check_roundtrip(ser, func)                         2806            self._check_roundtrip(ser, func)
2807                                                                     2807
2808            ser = Series(values, [datetime.datetime(                 2808            ser = Series(values, [datetime.datetime(
2809                2012, 1, 1), datetime.datetime(2012, 1, 2)])         2809                2012, 1, 1), datetime.datetime(2012, 1, 2)])
2810            self._check_roundtrip(ser, func)                         2810            self._check_roundtrip(ser, func)
2811                                                                     2811
2812    def test_timeseries_preepoch(self):                              2812    def test_timeseries_preepoch(self):
2813                                                                     2813
2814        if sys.version_info[0] == 2 and sys.version_info[1] < 7:     2814        if sys.version_info[0] == 2 and sys.version_info[1] < 7:
2815            pytest.skip("won't work on Python < 2.7")                2815            pytest.skip("won't work on Python < 2.7")
2816                                                                     2816
2817        dr = bdate_range('1/1/1940', '1/1/1960')                     2817        dr = bdate_range('1/1/1940', '1/1/1960')
2818        ts = Series(np.random.randn(len(dr)), index=dr)              2818        ts = Series(np.random.randn(len(dr)), index=dr)
2819        try:                                                         2819        try:
2820            self._check_roundtrip(ts, tm.assert_series_equal)        2820            self._check_roundtrip(ts, tm.assert_series_equal)
2821        except OverflowError:                                        2821        except OverflowError:
2822            pytest.skip('known failer on some windows platforms')    2822            pytest.skip('known failer on some windows platforms')
2823                                                                     2823
2824    def test_frame(self):                                            2824    def test_frame(self):
2825                                                                     2825
2826        df = tm.makeDataFrame()                                      2826        df = tm.makeDataFrame()
2827                                                                     2827
2828        # put in some random NAs                                     2828        # put in some random NAs
2829        df.values[0, 0] = np.nan                                     2829        df.values[0, 0] = np.nan
2830        df.values[5, 3] = np.nan                                     2830        df.values[5, 3] = np.nan
2831                                                                     2831
2832        self._check_roundtrip_table(df, tm.assert_frame_equal)       2832        self._check_roundtrip_table(df, tm.assert_frame_equal)
2833        self._check_roundtrip(df, tm.assert_frame_equal)             2833        self._check_roundtrip(df, tm.assert_frame_equal)
2834                                                                     2834
2835        if not skip_compression:                                     2835        if not skip_compression:
2836            self._check_roundtrip_table(df, tm.assert_frame_equal,   2836            self._check_roundtrip_table(df, tm.assert_frame_equal,
2837                                        compression=True)            2837                                        compression=True)
2838            self._check_roundtrip(df, tm.assert_frame_equal,         2838            self._check_roundtrip(df, tm.assert_frame_equal,
2839                                  compression=True)                  2839                                  compression=True)
2840                                                                     2840
2841        tdf = tm.makeTimeDataFrame()                                 2841        tdf = tm.makeTimeDataFrame()
2842        self._check_roundtrip(tdf, tm.assert_frame_equal)            2842        self._check_roundtrip(tdf, tm.assert_frame_equal)
2843                                                                     2843
2844        if not skip_compression:                                     2844        if not skip_compression:
2845            self._check_roundtrip(tdf, tm.assert_frame_equal,        2845            self._check_roundtrip(tdf, tm.assert_frame_equal,
2846                                  compression=True)                  2846                                  compression=True)
2847                                                                     2847
2848        with ensure_clean_store(self.path) as store:                 2848        with ensure_clean_store(self.path) as store:
2849            # not consolidated                                       2849            # not consolidated
2850            df['foo'] = np.random.randn(len(df))                     2850            df['foo'] = np.random.randn(len(df))
2851            store['df'] = df                                         2851            store['df'] = df
2852            recons = store['df']                                     2852            recons = store['df']
2853            assert recons._data.is_consolidated()                    2853            assert recons._data.is_consolidated()
2854                                                                     2854
2855        # empty                                                      2855        # empty
2856        self._check_roundtrip(df[:0], tm.assert_frame_equal)         2856        self._check_roundtrip(df[:0], tm.assert_frame_equal)
2857                                                                     2857
2858    def test_empty_series_frame(self):                               2858    def test_empty_series_frame(self):
2859        s0 = Series()                                                2859        s0 = Series()
2860        s1 = Series(name='myseries')                                 2860        s1 = Series(name='myseries')
2861        df0 = DataFrame()                                            2861        df0 = DataFrame()
2862        df1 = DataFrame(index=['a', 'b', 'c'])                       2862        df1 = DataFrame(index=['a', 'b', 'c'])
2863        df2 = DataFrame(columns=['d', 'e', 'f'])                     2863        df2 = DataFrame(columns=['d', 'e', 'f'])
2864                                                                     2864
2865        self._check_roundtrip(s0, tm.assert_series_equal)            2865        self._check_roundtrip(s0, tm.assert_series_equal)
2866        self._check_roundtrip(s1, tm.assert_series_equal)            2866        self._check_roundtrip(s1, tm.assert_series_equal)
2867        self._check_roundtrip(df0, tm.assert_frame_equal)            2867        self._check_roundtrip(df0, tm.assert_frame_equal)
2868        self._check_roundtrip(df1, tm.assert_frame_equal)            2868        self._check_roundtrip(df1, tm.assert_frame_equal)
2869        self._check_roundtrip(df2, tm.assert_frame_equal)            2869        self._check_roundtrip(df2, tm.assert_frame_equal)
2870                                                                     2870
2871    def test_empty_series(self):                                     2871    def test_empty_series(self):
2872        for dtype in [np.int64, np.float64, np.object, 'm8[ns]', 'M82872        for dtype in [np.int64, np.float64, np.object, 'm8[ns]', 'M8
                                                                 [ns]']:                                                                 [ns]']:
2873            s = Series(dtype=dtype)                                  2873            s = Series(dtype=dtype)
2874            self._check_roundtrip(s, tm.assert_series_equal)         2874            self._check_roundtrip(s, tm.assert_series_equal)
2875                                                                     2875
2876    def test_can_serialize_dates(self):                              2876    def test_can_serialize_dates(self):
2877                                                                     2877
2878        rng = [x.date() for x in bdate_range('1/1/2000', '1/30/2000'2878        rng = [x.date() for x in bdate_range('1/1/2000', '1/30/2000'
                                                                      )]                                                                      )]
2879        frame = DataFrame(np.random.randn(len(rng), 4), index=rng)   2879        frame = DataFrame(np.random.randn(len(rng), 4), index=rng)
2880                                                                     2880
2881        self._check_roundtrip(frame, tm.assert_frame_equal)          2881        self._check_roundtrip(frame, tm.assert_frame_equal)
2882                                                                     2882
2883    def test_store_hierarchical(self):                               2883    def test_store_hierarchical(self):
2884        index = MultiIndex(levels=[['foo', 'bar', 'baz', 'qux'],     2884        index = MultiIndex(levels=[['foo', 'bar', 'baz', 'qux'],
2885                                   ['one', 'two', 'three']],         2885                                   ['one', 'two', 'three']],
2886                           labels=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3],   2886                           labels=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3],
2887                                   [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],  2887                                   [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],
2888                           names=['foo', 'bar'])                     2888                           names=['foo', 'bar'])
2889        frame = DataFrame(np.random.randn(10, 3), index=index,       2889        frame = DataFrame(np.random.randn(10, 3), index=index,
2890                          columns=['A', 'B', 'C'])                   2890                          columns=['A', 'B', 'C'])
2891                                                                     2891
2892        self._check_roundtrip(frame, tm.assert_frame_equal)          2892        self._check_roundtrip(frame, tm.assert_frame_equal)
2893        self._check_roundtrip(frame.T, tm.assert_frame_equal)        2893        self._check_roundtrip(frame.T, tm.assert_frame_equal)
2894        self._check_roundtrip(frame['A'], tm.assert_series_equal)    2894        self._check_roundtrip(frame['A'], tm.assert_series_equal)
2895                                                                     2895
2896        # check that the names are stored                            2896        # check that the names are stored
2897        with ensure_clean_store(self.path) as store:                 2897        with ensure_clean_store(self.path) as store:
2898            store['frame'] = frame                                   2898            store['frame'] = frame
2899            recons = store['frame']                                  2899            recons = store['frame']
2900            tm.assert_frame_equal(recons, frame)                     2900            tm.assert_frame_equal(recons, frame)
2901                                                                     2901
2902    def test_store_index_name(self):                                 2902    def test_store_index_name(self):
2903        df = tm.makeDataFrame()                                      2903        df = tm.makeDataFrame()
2904        df.index.name = 'foo'                                        2904        df.index.name = 'foo'
2905                                                                     2905
2906        with ensure_clean_store(self.path) as store:                 2906        with ensure_clean_store(self.path) as store:
2907            store['frame'] = df                                      2907            store['frame'] = df
2908            recons = store['frame']                                  2908            recons = store['frame']
2909            tm.assert_frame_equal(recons, df)                        2909            tm.assert_frame_equal(recons, df)
2910                                                                     2910
2911    def test_store_index_name_with_tz(self):                         2911    def test_store_index_name_with_tz(self):
2912        # GH 13884                                                   2912        # GH 13884
2913        df = pd.DataFrame({'A': [1, 2]})                             2913        df = pd.DataFrame({'A': [1, 2]})
2914        df.index = pd.DatetimeIndex([1234567890123456787, 12345678902914        df.index = pd.DatetimeIndex([1234567890123456787, 1234567890
                                                             123456788])                                                             123456788])
2915        df.index = df.index.tz_localize('UTC')                       2915        df.index = df.index.tz_localize('UTC')
2916        df.index.name = 'foo'                                        2916        df.index.name = 'foo'
2917                                                                     2917
2918        with ensure_clean_store(self.path) as store:                 2918        with ensure_clean_store(self.path) as store:
2919            store.put('frame', df, format='table')                   2919            store.put('frame', df, format='table')
2920            recons = store['frame']                                  2920            recons = store['frame']
2921            tm.assert_frame_equal(recons, df)                        2921            tm.assert_frame_equal(recons, df)
2922                                                                     2922
2923    @pytest.mark.parametrize('table_format', ['table', 'fixed'])     2923    @pytest.mark.parametrize('table_format', ['table', 'fixed'])
2924    def test_store_index_name_numpy_str(self, table_format):         2924    def test_store_index_name_numpy_str(self, table_format):
2925        # GH #13492                                                  2925        # GH #13492
2926        idx = pd.Index(pd.to_datetime([datetime.date(2000, 1, 1),    2926        idx = pd.Index(pd.to_datetime([datetime.date(2000, 1, 1),
2927                                       datetime.date(2000, 1, 2)]),  2927                                       datetime.date(2000, 1, 2)]),
2928                       name=u('cols\u05d2'))                         2928                       name=u('cols\u05d2'))
2929        idx1 = pd.Index(pd.to_datetime([datetime.date(2010, 1, 1),   2929        idx1 = pd.Index(pd.to_datetime([datetime.date(2010, 1, 1),
2930                                        datetime.date(2010, 1, 2)]), 2930                                        datetime.date(2010, 1, 2)]),
2931                        name=u('rows\u05d0'))                        2931                        name=u('rows\u05d0'))
2932        df = pd.DataFrame(np.arange(4).reshape(2, 2), columns=idx, i2932        df = pd.DataFrame(np.arange(4).reshape(2, 2), columns=idx, i
                                                              ndex=idx1)                                                              ndex=idx1)
2933                                                                     2933
2934        # This used to fail, returning numpy strings instead of pyth2934        # This used to fail, returning numpy strings instead of pyth
                                                             on strings.                                                             on strings.
2935        with ensure_clean_path(self.path) as path:                   2935        with ensure_clean_path(self.path) as path:
2936            df.to_hdf(path, 'df', format=table_format)               2936            df.to_hdf(path, 'df', format=table_format)
2937            df2 = read_hdf(path, 'df')                               2937            df2 = read_hdf(path, 'df')
2938                                                                     2938
2939            assert_frame_equal(df, df2, check_names=True)            2939            assert_frame_equal(df, df2, check_names=True)
2940                                                                     2940
2941            assert type(df2.index.name) == text_type                 2941            assert type(df2.index.name) == text_type
2942            assert type(df2.columns.name) == text_type               2942            assert type(df2.columns.name) == text_type
2943                                                                     2943
2944    def test_store_series_name(self):                                2944    def test_store_series_name(self):
2945        df = tm.makeDataFrame()                                      2945        df = tm.makeDataFrame()
2946        series = df['A']                                             2946        series = df['A']
2947                                                                     2947
2948        with ensure_clean_store(self.path) as store:                 2948        with ensure_clean_store(self.path) as store:
2949            store['series'] = series                                 2949            store['series'] = series
2950            recons = store['series']                                 2950            recons = store['series']
2951            tm.assert_series_equal(recons, series)                   2951            tm.assert_series_equal(recons, series)
2952                                                                     2952
2953    def test_store_mixed(self):                                      2953    def test_store_mixed(self):
2954                                                                     2954
2955        def _make_one():                                             2955        def _make_one():
2956            df = tm.makeDataFrame()                                  2956            df = tm.makeDataFrame()
2957            df['obj1'] = 'foo'                                       2957            df['obj1'] = 'foo'
2958            df['obj2'] = 'bar'                                       2958            df['obj2'] = 'bar'
2959            df['bool1'] = df['A'] > 0                                2959            df['bool1'] = df['A'] > 0
2960            df['bool2'] = df['B'] > 0                                2960            df['bool2'] = df['B'] > 0
2961            df['int1'] = 1                                           2961            df['int1'] = 1
2962            df['int2'] = 2                                           2962            df['int2'] = 2
2963            return df._consolidate()                                 2963            return df._consolidate()
2964                                                                     2964
2965        df1 = _make_one()                                            2965        df1 = _make_one()
2966        df2 = _make_one()                                            2966        df2 = _make_one()
2967                                                                     2967
2968        self._check_roundtrip(df1, tm.assert_frame_equal)            2968        self._check_roundtrip(df1, tm.assert_frame_equal)
2969        self._check_roundtrip(df2, tm.assert_frame_equal)            2969        self._check_roundtrip(df2, tm.assert_frame_equal)
2970                                                                     2970
2971        with ensure_clean_store(self.path) as store:                 2971        with ensure_clean_store(self.path) as store:
2972            store['obj'] = df1                                       2972            store['obj'] = df1
2973            tm.assert_frame_equal(store['obj'], df1)                 2973            tm.assert_frame_equal(store['obj'], df1)
2974            store['obj'] = df2                                       2974            store['obj'] = df2
2975            tm.assert_frame_equal(store['obj'], df2)                 2975            tm.assert_frame_equal(store['obj'], df2)
2976                                                                     2976
2977        # check that can store Series of all of these types          2977        # check that can store Series of all of these types
2978        self._check_roundtrip(df1['obj1'], tm.assert_series_equal)   2978        self._check_roundtrip(df1['obj1'], tm.assert_series_equal)
2979        self._check_roundtrip(df1['bool1'], tm.assert_series_equal)  2979        self._check_roundtrip(df1['bool1'], tm.assert_series_equal)
2980        self._check_roundtrip(df1['int1'], tm.assert_series_equal)   2980        self._check_roundtrip(df1['int1'], tm.assert_series_equal)
2981                                                                     2981
2982        if not skip_compression:                                     2982        if not skip_compression:
2983            self._check_roundtrip(df1['obj1'], tm.assert_series_equa2983            self._check_roundtrip(df1['obj1'], tm.assert_series_equa
                                                                      l,                                                                      l,
2984                                  compression=True)                  2984                                  compression=True)
2985            self._check_roundtrip(df1['bool1'], tm.assert_series_equ2985            self._check_roundtrip(df1['bool1'], tm.assert_series_equ
                                                                     al,                                                                     al,
2986                                  compression=True)                  2986                                  compression=True)
2987            self._check_roundtrip(df1['int1'], tm.assert_series_equa2987            self._check_roundtrip(df1['int1'], tm.assert_series_equa
                                                                      l,                                                                      l,
2988                                  compression=True)                  2988                                  compression=True)
2989            self._check_roundtrip(df1, tm.assert_frame_equal,        2989            self._check_roundtrip(df1, tm.assert_frame_equal,
2990                                  compression=True)                  2990                                  compression=True)
2991                                                                     2991
2992    def test_wide(self):                                             2992    def test_wide(self):
2993                                                                     2993
2994        with catch_warnings(record=True):                            2994        with catch_warnings(record=True):
2995            wp = tm.makePanel()                                      2995            wp = tm.makePanel()
2996            self._check_roundtrip(wp, assert_panel_equal)            2996            self._check_roundtrip(wp, assert_panel_equal)
2997                                                                     2997
2998    def test_select_with_dups(self):                                 2998    def test_select_with_dups(self):
2999                                                                     2999
3000        # single dtypes                                              3000        # single dtypes
3001        df = DataFrame(np.random.randn(10, 4), columns=['A', 'A', 'B3001        df = DataFrame(np.random.randn(10, 4), columns=['A', 'A', 'B
                                                                ', 'B'])                                                                ', 'B'])
3002        df.index = date_range('20130101 9:30', periods=10, freq='T') 3002        df.index = date_range('20130101 9:30', periods=10, freq='T')
3003                                                                     3003
3004        with ensure_clean_store(self.path) as store:                 3004        with ensure_clean_store(self.path) as store:
3005            store.append('df', df)                                   3005            store.append('df', df)
3006                                                                     3006
3007            result = store.select('df')                              3007            result = store.select('df')
3008            expected = df                                            3008            expected = df
3009            assert_frame_equal(result, expected, by_blocks=True)     3009            assert_frame_equal(result, expected, by_blocks=True)
3010                                                                     3010
3011            result = store.select('df', columns=df.columns)          3011            result = store.select('df', columns=df.columns)
3012            expected = df                                            3012            expected = df
3013            assert_frame_equal(result, expected, by_blocks=True)     3013            assert_frame_equal(result, expected, by_blocks=True)
3014                                                                     3014
3015            result = store.select('df', columns=['A'])               3015            result = store.select('df', columns=['A'])
3016            expected = df.loc[:, ['A']]                              3016            expected = df.loc[:, ['A']]
3017            assert_frame_equal(result, expected)                     3017            assert_frame_equal(result, expected)
3018                                                                     3018
3019        # dups accross dtypes                                        3019        # dups accross dtypes
3020        df = concat([DataFrame(np.random.randn(10, 4),               3020        df = concat([DataFrame(np.random.randn(10, 4),
3021                               columns=['A', 'A', 'B', 'B']),        3021                               columns=['A', 'A', 'B', 'B']),
3022                     DataFrame(np.random.randint(0, 10, size=20)     3022                     DataFrame(np.random.randint(0, 10, size=20)
3023                               .reshape(10, 2),                      3023                               .reshape(10, 2),
3024                               columns=['A', 'C'])],                 3024                               columns=['A', 'C'])],
3025                    axis=1)                                          3025                    axis=1)
3026        df.index = date_range('20130101 9:30', periods=10, freq='T') 3026        df.index = date_range('20130101 9:30', periods=10, freq='T')
3027                                                                     3027
3028        with ensure_clean_store(self.path) as store:                 3028        with ensure_clean_store(self.path) as store:
3029            store.append('df', df)                                   3029            store.append('df', df)
3030                                                                     3030
3031            result = store.select('df')                              3031            result = store.select('df')
3032            expected = df                                            3032            expected = df
3033            assert_frame_equal(result, expected, by_blocks=True)     3033            assert_frame_equal(result, expected, by_blocks=True)
3034                                                                     3034
3035            result = store.select('df', columns=df.columns)          3035            result = store.select('df', columns=df.columns)
3036            expected = df                                            3036            expected = df
3037            assert_frame_equal(result, expected, by_blocks=True)     3037            assert_frame_equal(result, expected, by_blocks=True)
3038                                                                     3038
3039            expected = df.loc[:, ['A']]                              3039            expected = df.loc[:, ['A']]
3040            result = store.select('df', columns=['A'])               3040            result = store.select('df', columns=['A'])
3041            assert_frame_equal(result, expected, by_blocks=True)     3041            assert_frame_equal(result, expected, by_blocks=True)
3042                                                                     3042
3043            expected = df.loc[:, ['B', 'A']]                         3043            expected = df.loc[:, ['B', 'A']]
3044            result = store.select('df', columns=['B', 'A'])          3044            result = store.select('df', columns=['B', 'A'])
3045            assert_frame_equal(result, expected, by_blocks=True)     3045            assert_frame_equal(result, expected, by_blocks=True)
3046                                                                     3046
3047        # duplicates on both index and columns                       3047        # duplicates on both index and columns
3048        with ensure_clean_store(self.path) as store:                 3048        with ensure_clean_store(self.path) as store:
3049            store.append('df', df)                                   3049            store.append('df', df)
3050            store.append('df', df)                                   3050            store.append('df', df)
3051                                                                     3051
3052            expected = df.loc[:, ['B', 'A']]                         3052            expected = df.loc[:, ['B', 'A']]
3053            expected = concat([expected, expected])                  3053            expected = concat([expected, expected])
3054            result = store.select('df', columns=['B', 'A'])          3054            result = store.select('df', columns=['B', 'A'])
3055            assert_frame_equal(result, expected, by_blocks=True)     3055            assert_frame_equal(result, expected, by_blocks=True)
3056                                                                     3056
3057    def test_wide_table_dups(self):                                  3057    def test_wide_table_dups(self):
3058        with ensure_clean_store(self.path) as store:                 3058        with ensure_clean_store(self.path) as store:
3059            with catch_warnings(record=True):                        3059            with catch_warnings(record=True):
3060                                                                     3060
3061                wp = tm.makePanel()                                  3061                wp = tm.makePanel()
3062                store.put('panel', wp, format='table')               3062                store.put('panel', wp, format='table')
3063                store.put('panel', wp, format='table', append=True)  3063                store.put('panel', wp, format='table', append=True)
3064                                                                     3064
3065                recons = store['panel']                              3065                recons = store['panel']
3066                                                                     3066
3067                assert_panel_equal(recons, wp)                       3067                assert_panel_equal(recons, wp)
3068                                                                     3068
3069    def test_long(self):                                             3069    def test_long(self):
3070        def _check(left, right):                                     3070        def _check(left, right):
3071            assert_panel_equal(left.to_panel(), right.to_panel())    3071            assert_panel_equal(left.to_panel(), right.to_panel())
3072                                                                     3072
3073        with catch_warnings(record=True):                            3073        with catch_warnings(record=True):
3074            wp = tm.makePanel()                                      3074            wp = tm.makePanel()
3075            self._check_roundtrip(wp.to_frame(), _check)             3075            self._check_roundtrip(wp.to_frame(), _check)
3076                                                                     3076
3077    def test_longpanel(self):                                        3077    def test_longpanel(self):
3078        pass                                                         3078        pass
3079                                                                     3079
3080    def test_overwrite_node(self):                                   3080    def test_overwrite_node(self):
3081                                                                     3081
3082        with ensure_clean_store(self.path) as store:                 3082        with ensure_clean_store(self.path) as store:
3083            store['a'] = tm.makeTimeDataFrame()                      3083            store['a'] = tm.makeTimeDataFrame()
3084            ts = tm.makeTimeSeries()                                 3084            ts = tm.makeTimeSeries()
3085            store['a'] = ts                                          3085            store['a'] = ts
3086                                                                     3086
3087            tm.assert_series_equal(store['a'], ts)                   3087            tm.assert_series_equal(store['a'], ts)
3088                                                                     3088
3089    def test_sparse_with_compression(self):                          3089    def test_sparse_with_compression(self):
3090                                                                     3090
3091        # GH 2931                                                    3091        # GH 2931
3092                                                                     3092
3093        # make sparse dataframe                                      3093        # make sparse dataframe
3094        arr = np.random.binomial(n=1, p=.01, size=(1000, 10))        3094        arr = np.random.binomial(n=1, p=.01, size=(1000, 10))
3095        df = DataFrame(arr).to_sparse(fill_value=0)                  3095        df = DataFrame(arr).to_sparse(fill_value=0)
3096                                                                     3096
3097        # case 1: store uncompressed                                 3097        # case 1: store uncompressed
3098        self._check_double_roundtrip(df, tm.assert_frame_equal,      3098        self._check_double_roundtrip(df, tm.assert_frame_equal,
3099                                     compression=False,              3099                                     compression=False,
3100                                     check_frame_type=True)          3100                                     check_frame_type=True)
3101                                                                     3101
3102        # case 2: store compressed (works)                           3102        # case 2: store compressed (works)
3103        self._check_double_roundtrip(df, tm.assert_frame_equal,      3103        self._check_double_roundtrip(df, tm.assert_frame_equal,
3104                                     compression='zlib',             3104                                     compression='zlib',
3105                                     check_frame_type=True)          3105                                     check_frame_type=True)
3106                                                                     3106
3107        # set one series to be completely sparse                     3107        # set one series to be completely sparse
3108        df[0] = np.zeros(1000)                                       3108        df[0] = np.zeros(1000)
3109                                                                     3109
3110        # case 3: store df with completely sparse series uncompressed3110        # case 3: store df with completely sparse series uncompressed
3111        self._check_double_roundtrip(df, tm.assert_frame_equal,      3111        self._check_double_roundtrip(df, tm.assert_frame_equal,
3112                                     compression=False,              3112                                     compression=False,
3113                                     check_frame_type=True)          3113                                     check_frame_type=True)
3114                                                                     3114
3115        # case 4: try storing df with completely sparse series compr3115        # case 4: try storing df with completely sparse series compr
                                                                   essed                                                                   essed
3116        # (fails)                                                    3116        # (fails)
3117        self._check_double_roundtrip(df, tm.assert_frame_equal,      3117        self._check_double_roundtrip(df, tm.assert_frame_equal,
3118                                     compression='zlib',             3118                                     compression='zlib',
3119                                     check_frame_type=True)          3119                                     check_frame_type=True)
3120                                                                     3120
3121    def test_select(self):                                           3121    def test_select(self):
3122                                                                     3122
3123        with ensure_clean_store(self.path) as store:                 3123        with ensure_clean_store(self.path) as store:
3124                                                                     3124
3125            with catch_warnings(record=True):                        3125            with catch_warnings(record=True):
3126                wp = tm.makePanel()                                  3126                wp = tm.makePanel()
3127                                                                     3127
3128                # put/select ok                                      3128                # put/select ok
3129                _maybe_remove(store, 'wp')                           3129                _maybe_remove(store, 'wp')
3130                store.put('wp', wp, format='table')                  3130                store.put('wp', wp, format='table')
3131                store.select('wp')                                   3131                store.select('wp')
3132                                                                     3132
3133                # non-table ok (where = None)                        3133                # non-table ok (where = None)
3134                _maybe_remove(store, 'wp')                           3134                _maybe_remove(store, 'wp')
3135                store.put('wp2', wp)                                 3135                store.put('wp2', wp)
3136                store.select('wp2')                                  3136                store.select('wp2')
3137                                                                     3137
3138                # selection on the non-indexable with a large number3138                # selection on the non-indexable with a large number
                                                              of columns                                                              of columns
3139                wp = Panel(np.random.randn(100, 100, 100),           3139                wp = Panel(np.random.randn(100, 100, 100),
3140                           items=['Item%03d' % i for i in range(100)3140                           items=['Item%03d' % i for i in range(100)
                                                                      ],                                                                      ],
3141                           major_axis=date_range('1/1/2000', periods3141                           major_axis=date_range('1/1/2000', periods
                                                                  =100),                                                                  =100),
3142                           minor_axis=['E%03d' % i for i in range(103142                           minor_axis=['E%03d' % i for i in range(10
                                                                    0)])                                                                    0)])
3143                                                                     3143
3144                _maybe_remove(store, 'wp')                           3144                _maybe_remove(store, 'wp')
3145                store.append('wp', wp)                               3145                store.append('wp', wp)
3146                items = ['Item%03d' % i for i in range(80)]          3146                items = ['Item%03d' % i for i in range(80)]
3147                result = store.select('wp', 'items=items')           3147                result = store.select('wp', 'items=items')
3148                expected = wp.reindex(items=items)                   3148                expected = wp.reindex(items=items)
3149                assert_panel_equal(expected, result)                 3149                assert_panel_equal(expected, result)
3150                                                                     3150
3151                # selectin non-table with a where                    3151                # selectin non-table with a where
3152                # pytest.raises(ValueError, store.select,            3152                # pytest.raises(ValueError, store.select,
3153                #                  'wp2', ('column', ['A', 'D']))    3153                #                  'wp2', ('column', ['A', 'D']))
3154                                                                     3154
3155                # select with columns=                               3155                # select with columns=
3156                df = tm.makeTimeDataFrame()                          3156                df = tm.makeTimeDataFrame()
3157                _maybe_remove(store, 'df')                           3157                _maybe_remove(store, 'df')
3158                store.append('df', df)                               3158                store.append('df', df)
3159                result = store.select('df', columns=['A', 'B'])      3159                result = store.select('df', columns=['A', 'B'])
3160                expected = df.reindex(columns=['A', 'B'])            3160                expected = df.reindex(columns=['A', 'B'])
3161                tm.assert_frame_equal(expected, result)              3161                tm.assert_frame_equal(expected, result)
3162                                                                     3162
3163                # equivalentsly                                      3163                # equivalentsly
3164                result = store.select('df', [("columns=['A', 'B']")])3164                result = store.select('df', [("columns=['A', 'B']")])
3165                expected = df.reindex(columns=['A', 'B'])            3165                expected = df.reindex(columns=['A', 'B'])
3166                tm.assert_frame_equal(expected, result)              3166                tm.assert_frame_equal(expected, result)
3167                                                                     3167
3168                # with a data column                                 3168                # with a data column
3169                _maybe_remove(store, 'df')                           3169                _maybe_remove(store, 'df')
3170                store.append('df', df, data_columns=['A'])           3170                store.append('df', df, data_columns=['A'])
3171                result = store.select('df', ['A > 0'], columns=['A',3171                result = store.select('df', ['A > 0'], columns=['A',
                                                                   'B'])                                                                   'B'])
3172                expected = df[df.A > 0].reindex(columns=['A', 'B'])  3172                expected = df[df.A > 0].reindex(columns=['A', 'B'])
3173                tm.assert_frame_equal(expected, result)              3173                tm.assert_frame_equal(expected, result)
3174                                                                     3174
3175                # all a data columns                                 3175                # all a data columns
3176                _maybe_remove(store, 'df')                           3176                _maybe_remove(store, 'df')
3177                store.append('df', df, data_columns=True)            3177                store.append('df', df, data_columns=True)
3178                result = store.select('df', ['A > 0'], columns=['A',3178                result = store.select('df', ['A > 0'], columns=['A',
                                                                   'B'])                                                                   'B'])
3179                expected = df[df.A > 0].reindex(columns=['A', 'B'])  3179                expected = df[df.A > 0].reindex(columns=['A', 'B'])
3180                tm.assert_frame_equal(expected, result)              3180                tm.assert_frame_equal(expected, result)
3181                                                                     3181
3182                # with a data column, but different columns          3182                # with a data column, but different columns
3183                _maybe_remove(store, 'df')                           3183                _maybe_remove(store, 'df')
3184                store.append('df', df, data_columns=['A'])           3184                store.append('df', df, data_columns=['A'])
3185                result = store.select('df', ['A > 0'], columns=['C',3185                result = store.select('df', ['A > 0'], columns=['C',
                                                                   'D'])                                                                   'D'])
3186                expected = df[df.A > 0].reindex(columns=['C', 'D'])  3186                expected = df[df.A > 0].reindex(columns=['C', 'D'])
3187                tm.assert_frame_equal(expected, result)              3187                tm.assert_frame_equal(expected, result)
3188                                                                     3188
3189    def test_select_dtypes(self):                                    3189    def test_select_dtypes(self):
3190                                                                     3190
3191        with ensure_clean_store(self.path) as store:                 3191        with ensure_clean_store(self.path) as store:
3192            # with a Timestamp data column (GH #2637)                3192            # with a Timestamp data column (GH #2637)
3193            df = DataFrame(dict(                                     3193            df = DataFrame(dict(
3194                ts=bdate_range('2012-01-01', periods=300),           3194                ts=bdate_range('2012-01-01', periods=300),
3195                A=np.random.randn(300)))                             3195                A=np.random.randn(300)))
3196            _maybe_remove(store, 'df')                               3196            _maybe_remove(store, 'df')
3197            store.append('df', df, data_columns=['ts', 'A'])         3197            store.append('df', df, data_columns=['ts', 'A'])
3198                                                                     3198
3199            result = store.select('df', "ts>=Timestamp('2012-02-01')3199            result = store.select('df', "ts>=Timestamp('2012-02-01')
                                                                      ")                                                                      ")
3200            expected = df[df.ts >= Timestamp('2012-02-01')]          3200            expected = df[df.ts >= Timestamp('2012-02-01')]
3201            tm.assert_frame_equal(expected, result)                  3201            tm.assert_frame_equal(expected, result)
3202                                                                     3202
3203            # bool columns (GH #2849)                                3203            # bool columns (GH #2849)
3204            df = DataFrame(np.random.randn(5, 2), columns=['A', 'B'])3204            df = DataFrame(np.random.randn(5, 2), columns=['A', 'B'])
3205            df['object'] = 'foo'                                     3205            df['object'] = 'foo'
3206            df.loc[4:5, 'object'] = 'bar'                            3206            df.loc[4:5, 'object'] = 'bar'
3207            df['boolv'] = df['A'] > 0                                3207            df['boolv'] = df['A'] > 0
3208            _maybe_remove(store, 'df')                               3208            _maybe_remove(store, 'df')
3209            store.append('df', df, data_columns=True)                3209            store.append('df', df, data_columns=True)
3210                                                                     3210
3211            expected = (df[df.boolv == True]  # noqa                 3211            expected = (df[df.boolv == True]  # noqa
3212                        .reindex(columns=['A', 'boolv']))            3212                        .reindex(columns=['A', 'boolv']))
3213            for v in [True, 'true', 1]:                              3213            for v in [True, 'true', 1]:
3214                result = store.select('df', 'boolv == %s' % str(v),  3214                result = store.select('df', 'boolv == %s' % str(v),
3215                                      columns=['A', 'boolv'])        3215                                      columns=['A', 'boolv'])
3216                tm.assert_frame_equal(expected, result)              3216                tm.assert_frame_equal(expected, result)
3217                                                                     3217
3218            expected = (df[df.boolv == False]  # noqa                3218            expected = (df[df.boolv == False]  # noqa
3219                        .reindex(columns=['A', 'boolv']))            3219                        .reindex(columns=['A', 'boolv']))
3220            for v in [False, 'false', 0]:                            3220            for v in [False, 'false', 0]:
3221                result = store.select(                               3221                result = store.select(
3222                    'df', 'boolv == %s' % str(v), columns=['A', 'boo3222                    'df', 'boolv == %s' % str(v), columns=['A', 'boo
                                                                   lv'])                                                                   lv'])
3223                tm.assert_frame_equal(expected, result)              3223                tm.assert_frame_equal(expected, result)
3224                                                                     3224
3225            # integer index                                          3225            # integer index
3226            df = DataFrame(dict(A=np.random.rand(20), B=np.random.ra3226            df = DataFrame(dict(A=np.random.rand(20), B=np.random.ra
                                                                nd(20)))                                                                nd(20)))
3227            _maybe_remove(store, 'df_int')                           3227            _maybe_remove(store, 'df_int')
3228            store.append('df_int', df)                               3228            store.append('df_int', df)
3229            result = store.select(                                   3229            result = store.select(
3230                'df_int', "index<10 and columns=['A']")              3230                'df_int', "index<10 and columns=['A']")
3231            expected = df.reindex(index=list(df.index)[0:10], column3231            expected = df.reindex(index=list(df.index)[0:10], column
                                                                s=['A'])                                                                s=['A'])
3232            tm.assert_frame_equal(expected, result)                  3232            tm.assert_frame_equal(expected, result)
3233                                                                     3233
3234            # float index                                            3234            # float index
3235            df = DataFrame(dict(A=np.random.rand(                    3235            df = DataFrame(dict(A=np.random.rand(
3236                20), B=np.random.rand(20), index=np.arange(20, dtype3236                20), B=np.random.rand(20), index=np.arange(20, dtype
                                                                ='f8')))                                                                ='f8')))
3237            _maybe_remove(store, 'df_float')                         3237            _maybe_remove(store, 'df_float')
3238            store.append('df_float', df)                             3238            store.append('df_float', df)
3239            result = store.select(                                   3239            result = store.select(
3240                'df_float', "index<10.0 and columns=['A']")          3240                'df_float', "index<10.0 and columns=['A']")
3241            expected = df.reindex(index=list(df.index)[0:10], column3241            expected = df.reindex(index=list(df.index)[0:10], column
                                                                s=['A'])                                                                s=['A'])
3242            tm.assert_frame_equal(expected, result)                  3242            tm.assert_frame_equal(expected, result)
3243                                                                     3243
3244        with ensure_clean_store(self.path) as store:                 3244        with ensure_clean_store(self.path) as store:
3245                                                                     3245
3246            # floats w/o NaN                                         3246            # floats w/o NaN
3247            df = DataFrame(                                          3247            df = DataFrame(
3248                dict(cols=range(11), values=range(11)), dtype='float3248                dict(cols=range(11), values=range(11)), dtype='float
                                                                    64')                                                                    64')
3249            df['cols'] = (df['cols'] + 10).apply(str)                3249            df['cols'] = (df['cols'] + 10).apply(str)
3250                                                                     3250
3251            store.append('df1', df, data_columns=True)               3251            store.append('df1', df, data_columns=True)
3252            result = store.select(                                   3252            result = store.select(
3253                'df1', where='values>2.0')                           3253                'df1', where='values>2.0')
3254            expected = df[df['values'] > 2.0]                        3254            expected = df[df['values'] > 2.0]
3255            tm.assert_frame_equal(expected, result)                  3255            tm.assert_frame_equal(expected, result)
3256                                                                     3256
3257            # floats with NaN                                        3257            # floats with NaN
3258            df.iloc[0] = np.nan                                      3258            df.iloc[0] = np.nan
3259            expected = df[df['values'] > 2.0]                        3259            expected = df[df['values'] > 2.0]
3260                                                                     3260
3261            store.append('df2', df, data_columns=True, index=False)  3261            store.append('df2', df, data_columns=True, index=False)
3262            result = store.select(                                   3262            result = store.select(
3263                'df2', where='values>2.0')                           3263                'df2', where='values>2.0')
3264            tm.assert_frame_equal(expected, result)                  3264            tm.assert_frame_equal(expected, result)
3265                                                                     3265
3266            # https://github.com/PyTables/PyTables/issues/282        3266            # https://github.com/PyTables/PyTables/issues/282
3267            # bug in selection when 0th row has a np.nan and an index3267            # bug in selection when 0th row has a np.nan and an index
3268            # store.append('df3',df,data_columns=True)               3268            # store.append('df3',df,data_columns=True)
3269            # result = store.select(                                 3269            # result = store.select(
3270            #    'df3', where='values>2.0')                          3270            #    'df3', where='values>2.0')
3271            # tm.assert_frame_equal(expected, result)                3271            # tm.assert_frame_equal(expected, result)
3272                                                                     3272
3273            # not in first position float with NaN ok too            3273            # not in first position float with NaN ok too
3274            df = DataFrame(                                          3274            df = DataFrame(
3275                dict(cols=range(11), values=range(11)), dtype='float3275                dict(cols=range(11), values=range(11)), dtype='float
                                                                    64')                                                                    64')
3276            df['cols'] = (df['cols'] + 10).apply(str)                3276            df['cols'] = (df['cols'] + 10).apply(str)
3277                                                                     3277
3278            df.iloc[1] = np.nan                                      3278            df.iloc[1] = np.nan
3279            expected = df[df['values'] > 2.0]                        3279            expected = df[df['values'] > 2.0]
3280                                                                     3280
3281            store.append('df4', df, data_columns=True)               3281            store.append('df4', df, data_columns=True)
3282            result = store.select(                                   3282            result = store.select(
3283                'df4', where='values>2.0')                           3283                'df4', where='values>2.0')
3284            tm.assert_frame_equal(expected, result)                  3284            tm.assert_frame_equal(expected, result)
3285                                                                     3285
3286        # test selection with comparison against numpy scalar        3286        # test selection with comparison against numpy scalar
3287        # GH 11283                                                   3287        # GH 11283
3288        with ensure_clean_store(self.path) as store:                 3288        with ensure_clean_store(self.path) as store:
3289            df = tm.makeDataFrame()                                  3289            df = tm.makeDataFrame()
3290                                                                     3290
3291            expected = df[df['A'] > 0]                               3291            expected = df[df['A'] > 0]
3292                                                                     3292
3293            store.append('df', df, data_columns=True)                3293            store.append('df', df, data_columns=True)
3294            np_zero = np.float64(0)  # noqa                          3294            np_zero = np.float64(0)  # noqa
3295            result = store.select('df', where=["A>np_zero"])         3295            result = store.select('df', where=["A>np_zero"])
3296            tm.assert_frame_equal(expected, result)                  3296            tm.assert_frame_equal(expected, result)
3297                                                                     3297
3298    def test_select_with_many_inputs(self):                          3298    def test_select_with_many_inputs(self):
3299                                                                     3299
3300        with ensure_clean_store(self.path) as store:                 3300        with ensure_clean_store(self.path) as store:
3301                                                                     3301
3302            df = DataFrame(dict(ts=bdate_range('2012-01-01', periods3302            df = DataFrame(dict(ts=bdate_range('2012-01-01', periods
                                                                  =300),                                                                  =300),
3303                                A=np.random.randn(300),              3303                                A=np.random.randn(300),
3304                                B=range(300),                        3304                                B=range(300),
3305                                users=['a'] * 50 + ['b'] * 50 + ['c'3305                                users=['a'] * 50 + ['b'] * 50 + ['c'
                                                               ] * 100 +                                                               ] * 100 +
3306                                ['a%03d' % i for i in range(100)]))  3306                                ['a%03d' % i for i in range(100)]))
3307            _maybe_remove(store, 'df')                               3307            _maybe_remove(store, 'df')
3308            store.append('df', df, data_columns=['ts', 'A', 'B', 'us3308            store.append('df', df, data_columns=['ts', 'A', 'B', 'us
                                                                  ers'])                                                                  ers'])
3309                                                                     3309
3310            # regular select                                         3310            # regular select
3311            result = store.select('df', "ts>=Timestamp('2012-02-01')3311            result = store.select('df', "ts>=Timestamp('2012-02-01')
                                                                      ")                                                                      ")
3312            expected = df[df.ts >= Timestamp('2012-02-01')]          3312            expected = df[df.ts >= Timestamp('2012-02-01')]
3313            tm.assert_frame_equal(expected, result)                  3313            tm.assert_frame_equal(expected, result)
3314                                                                     3314
3315            # small selector                                         3315            # small selector
3316            result = store.select(                                   3316            result = store.select(
3317                'df',                                                3317                'df',
3318                "ts>=Timestamp('2012-02-01') & users=['a','b','c']") 3318                "ts>=Timestamp('2012-02-01') & users=['a','b','c']")
3319            expected = df[(df.ts >= Timestamp('2012-02-01')) &       3319            expected = df[(df.ts >= Timestamp('2012-02-01')) &
3320                          df.users.isin(['a', 'b', 'c'])]            3320                          df.users.isin(['a', 'b', 'c'])]
3321            tm.assert_frame_equal(expected, result)                  3321            tm.assert_frame_equal(expected, result)
3322                                                                     3322
3323            # big selector along the columns                         3323            # big selector along the columns
3324            selector = ['a', 'b', 'c'] + ['a%03d' % i for i in range3324            selector = ['a', 'b', 'c'] + ['a%03d' % i for i in range
                                                                   (60)]                                                                   (60)]
3325            result = store.select(                                   3325            result = store.select(
3326                'df',                                                3326                'df',
3327                "ts>=Timestamp('2012-02-01') and users=selector")    3327                "ts>=Timestamp('2012-02-01') and users=selector")
3328            expected = df[(df.ts >= Timestamp('2012-02-01')) &       3328            expected = df[(df.ts >= Timestamp('2012-02-01')) &
3329                          df.users.isin(selector)]                   3329                          df.users.isin(selector)]
3330            tm.assert_frame_equal(expected, result)                  3330            tm.assert_frame_equal(expected, result)
3331                                                                     3331
3332            selector = range(100, 200)                               3332            selector = range(100, 200)
3333            result = store.select('df', 'B=selector')                3333            result = store.select('df', 'B=selector')
3334            expected = df[df.B.isin(selector)]                       3334            expected = df[df.B.isin(selector)]
3335            tm.assert_frame_equal(expected, result)                  3335            tm.assert_frame_equal(expected, result)
3336            assert len(result) == 100                                3336            assert len(result) == 100
3337                                                                     3337
3338            # big selector along the index                           3338            # big selector along the index
3339            selector = Index(df.ts[0:100].values)                    3339            selector = Index(df.ts[0:100].values)
3340            result = store.select('df', 'ts=selector')               3340            result = store.select('df', 'ts=selector')
3341            expected = df[df.ts.isin(selector.values)]               3341            expected = df[df.ts.isin(selector.values)]
3342            tm.assert_frame_equal(expected, result)                  3342            tm.assert_frame_equal(expected, result)
3343            assert len(result) == 100                                3343            assert len(result) == 100
3344                                                                     3344
3345    def test_select_iterator(self):                                  3345    def test_select_iterator(self):
3346                                                                     3346
3347        # single table                                               3347        # single table
3348        with ensure_clean_store(self.path) as store:                 3348        with ensure_clean_store(self.path) as store:
3349                                                                     3349
3350            df = tm.makeTimeDataFrame(500)                           3350            df = tm.makeTimeDataFrame(500)
3351            _maybe_remove(store, 'df')                               3351            _maybe_remove(store, 'df')
3352            store.append('df', df)                                   3352            store.append('df', df)
3353                                                                     3353
3354            expected = store.select('df')                            3354            expected = store.select('df')
3355                                                                     3355
3356            results = [s for s in store.select('df', iterator=True)] 3356            results = [s for s in store.select('df', iterator=True)]
3357            result = concat(results)                                 3357            result = concat(results)
3358            tm.assert_frame_equal(expected, result)                  3358            tm.assert_frame_equal(expected, result)
3359                                                                     3359
3360            results = [s for s in store.select('df', chunksize=100)] 3360            results = [s for s in store.select('df', chunksize=100)]
3361            assert len(results) == 5                                 3361            assert len(results) == 5
3362            result = concat(results)                                 3362            result = concat(results)
3363            tm.assert_frame_equal(expected, result)                  3363            tm.assert_frame_equal(expected, result)
3364                                                                     3364
3365            results = [s for s in store.select('df', chunksize=150)] 3365            results = [s for s in store.select('df', chunksize=150)]
3366            result = concat(results)                                 3366            result = concat(results)
3367            tm.assert_frame_equal(result, expected)                  3367            tm.assert_frame_equal(result, expected)
3368                                                                     3368
3369        with ensure_clean_path(self.path) as path:                   3369        with ensure_clean_path(self.path) as path:
3370                                                                     3370
3371            df = tm.makeTimeDataFrame(500)                           3371            df = tm.makeTimeDataFrame(500)
3372            df.to_hdf(path, 'df_non_table')                          3372            df.to_hdf(path, 'df_non_table')
3373            pytest.raises(TypeError, read_hdf, path,                 3373            pytest.raises(TypeError, read_hdf, path,
3374                          'df_non_table', chunksize=100)             3374                          'df_non_table', chunksize=100)
3375            pytest.raises(TypeError, read_hdf, path,                 3375            pytest.raises(TypeError, read_hdf, path,
3376                          'df_non_table', iterator=True)             3376                          'df_non_table', iterator=True)
3377                                                                     3377
3378        with ensure_clean_path(self.path) as path:                   3378        with ensure_clean_path(self.path) as path:
3379                                                                     3379
3380            df = tm.makeTimeDataFrame(500)                           3380            df = tm.makeTimeDataFrame(500)
3381            df.to_hdf(path, 'df', format='table')                    3381            df.to_hdf(path, 'df', format='table')
3382                                                                     3382
3383            results = [s for s in read_hdf(path, 'df', chunksize=1003383            results = [s for s in read_hdf(path, 'df', chunksize=100
                                                                      )]                                                                      )]
3384            result = concat(results)                                 3384            result = concat(results)
3385                                                                     3385
3386            assert len(results) == 5                                 3386            assert len(results) == 5
3387            tm.assert_frame_equal(result, df)                        3387            tm.assert_frame_equal(result, df)
3388            tm.assert_frame_equal(result, read_hdf(path, 'df'))      3388            tm.assert_frame_equal(result, read_hdf(path, 'df'))
3389                                                                     3389
3390        # multiple                                                   3390        # multiple
3391                                                                     3391
3392        with ensure_clean_store(self.path) as store:                 3392        with ensure_clean_store(self.path) as store:
3393                                                                     3393
3394            df1 = tm.makeTimeDataFrame(500)                          3394            df1 = tm.makeTimeDataFrame(500)
3395            store.append('df1', df1, data_columns=True)              3395            store.append('df1', df1, data_columns=True)
3396            df2 = tm.makeTimeDataFrame(500).rename(                  3396            df2 = tm.makeTimeDataFrame(500).rename(
3397                columns=lambda x: "%s_2" % x)                        3397                columns=lambda x: "%s_2" % x)
3398            df2['foo'] = 'bar'                                       3398            df2['foo'] = 'bar'
3399            store.append('df2', df2)                                 3399            store.append('df2', df2)
3400                                                                     3400
3401            df = concat([df1, df2], axis=1)                          3401            df = concat([df1, df2], axis=1)
3402                                                                     3402
3403            # full selection                                         3403            # full selection
3404            expected = store.select_as_multiple(                     3404            expected = store.select_as_multiple(
3405                ['df1', 'df2'], selector='df1')                      3405                ['df1', 'df2'], selector='df1')
3406            results = [s for s in store.select_as_multiple(          3406            results = [s for s in store.select_as_multiple(
3407                ['df1', 'df2'], selector='df1', chunksize=150)]      3407                ['df1', 'df2'], selector='df1', chunksize=150)]
3408            result = concat(results)                                 3408            result = concat(results)
3409            tm.assert_frame_equal(expected, result)                  3409            tm.assert_frame_equal(expected, result)
3410                                                                     3410
3411    def test_select_iterator_complete_8014(self):                    3411    def test_select_iterator_complete_8014(self):
3412                                                                     3412
3413        # GH 8014                                                    3413        # GH 8014
3414        # using iterator and where clause                            3414        # using iterator and where clause
3415        chunksize = 1e4                                              3415        chunksize = 1e4
3416                                                                     3416
3417        # no iterator                                                3417        # no iterator
3418        with ensure_clean_store(self.path) as store:                 3418        with ensure_clean_store(self.path) as store:
3419                                                                     3419
3420            expected = tm.makeTimeDataFrame(100064, 'S')             3420            expected = tm.makeTimeDataFrame(100064, 'S')
3421            _maybe_remove(store, 'df')                               3421            _maybe_remove(store, 'df')
3422            store.append('df', expected)                             3422            store.append('df', expected)
3423                                                                     3423
3424            beg_dt = expected.index[0]                               3424            beg_dt = expected.index[0]
3425            end_dt = expected.index[-1]                              3425            end_dt = expected.index[-1]
3426                                                                     3426
3427            # select w/o iteration and no where clause works         3427            # select w/o iteration and no where clause works
3428            result = store.select('df')                              3428            result = store.select('df')
3429            tm.assert_frame_equal(expected, result)                  3429            tm.assert_frame_equal(expected, result)
3430                                                                     3430
3431            # select w/o iterator and where clause, single term, beg3431            # select w/o iterator and where clause, single term, beg
                                                                      in                                                                      in
3432            # of range, works                                        3432            # of range, works
3433            where = "index >= '%s'" % beg_dt                         3433            where = "index >= '%s'" % beg_dt
3434            result = store.select('df', where=where)                 3434            result = store.select('df', where=where)
3435            tm.assert_frame_equal(expected, result)                  3435            tm.assert_frame_equal(expected, result)
3436                                                                     3436
3437            # select w/o iterator and where clause, single term, end 3437            # select w/o iterator and where clause, single term, end
3438            # of range, works                                        3438            # of range, works
3439            where = "index <= '%s'" % end_dt                         3439            where = "index <= '%s'" % end_dt
3440            result = store.select('df', where=where)                 3440            result = store.select('df', where=where)
3441            tm.assert_frame_equal(expected, result)                  3441            tm.assert_frame_equal(expected, result)
3442                                                                     3442
3443            # select w/o iterator and where clause, inclusive range, 3443            # select w/o iterator and where clause, inclusive range,
3444            # works                                                  3444            # works
3445            where = "index >= '%s' & index <= '%s'" % (beg_dt, end_d3445            where = "index >= '%s' & index <= '%s'" % (beg_dt, end_d
                                                                      t)                                                                      t)
3446            result = store.select('df', where=where)                 3446            result = store.select('df', where=where)
3447            tm.assert_frame_equal(expected, result)                  3447            tm.assert_frame_equal(expected, result)
3448                                                                     3448
3449        # with iterator, full range                                  3449        # with iterator, full range
3450        with ensure_clean_store(self.path) as store:                 3450        with ensure_clean_store(self.path) as store:
3451                                                                     3451
3452            expected = tm.makeTimeDataFrame(100064, 'S')             3452            expected = tm.makeTimeDataFrame(100064, 'S')
3453            _maybe_remove(store, 'df')                               3453            _maybe_remove(store, 'df')
3454            store.append('df', expected)                             3454            store.append('df', expected)
3455                                                                     3455
3456            beg_dt = expected.index[0]                               3456            beg_dt = expected.index[0]
3457            end_dt = expected.index[-1]                              3457            end_dt = expected.index[-1]
3458                                                                     3458
3459            # select w/iterator and no where clause works            3459            # select w/iterator and no where clause works
3460            results = [s for s in store.select('df', chunksize=chunk3460            results = [s for s in store.select('df', chunksize=chunk
                                                                  size)]                                                                  size)]
3461            result = concat(results)                                 3461            result = concat(results)
3462            tm.assert_frame_equal(expected, result)                  3462            tm.assert_frame_equal(expected, result)
3463                                                                     3463
3464            # select w/iterator and where clause, single term, begin3464            # select w/iterator and where clause, single term, begin
                                                                of range                                                                of range
3465            where = "index >= '%s'" % beg_dt                         3465            where = "index >= '%s'" % beg_dt
3466            results = [s for s in store.select(                      3466            results = [s for s in store.select(
3467                'df', where=where, chunksize=chunksize)]             3467                'df', where=where, chunksize=chunksize)]
3468            result = concat(results)                                 3468            result = concat(results)
3469            tm.assert_frame_equal(expected, result)                  3469            tm.assert_frame_equal(expected, result)
3470                                                                     3470
3471            # select w/iterator and where clause, single term, end o3471            # select w/iterator and where clause, single term, end o
                                                                 f range                                                                 f range
3472            where = "index <= '%s'" % end_dt                         3472            where = "index <= '%s'" % end_dt
3473            results = [s for s in store.select(                      3473            results = [s for s in store.select(
3474                'df', where=where, chunksize=chunksize)]             3474                'df', where=where, chunksize=chunksize)]
3475            result = concat(results)                                 3475            result = concat(results)
3476            tm.assert_frame_equal(expected, result)                  3476            tm.assert_frame_equal(expected, result)
3477                                                                     3477
3478            # select w/iterator and where clause, inclusive range    3478            # select w/iterator and where clause, inclusive range
3479            where = "index >= '%s' & index <= '%s'" % (beg_dt, end_d3479            where = "index >= '%s' & index <= '%s'" % (beg_dt, end_d
                                                                      t)                                                                      t)
3480            results = [s for s in store.select(                      3480            results = [s for s in store.select(
3481                'df', where=where, chunksize=chunksize)]             3481                'df', where=where, chunksize=chunksize)]
3482            result = concat(results)                                 3482            result = concat(results)
3483            tm.assert_frame_equal(expected, result)                  3483            tm.assert_frame_equal(expected, result)
3484                                                                     3484
3485    def test_select_iterator_non_complete_8014(self):                3485    def test_select_iterator_non_complete_8014(self):
3486                                                                     3486
3487        # GH 8014                                                    3487        # GH 8014
3488        # using iterator and where clause                            3488        # using iterator and where clause
3489        chunksize = 1e4                                              3489        chunksize = 1e4
3490                                                                     3490
3491        # with iterator, non complete range                          3491        # with iterator, non complete range
3492        with ensure_clean_store(self.path) as store:                 3492        with ensure_clean_store(self.path) as store:
3493                                                                     3493
3494            expected = tm.makeTimeDataFrame(100064, 'S')             3494            expected = tm.makeTimeDataFrame(100064, 'S')
3495            _maybe_remove(store, 'df')                               3495            _maybe_remove(store, 'df')
3496            store.append('df', expected)                             3496            store.append('df', expected)
3497                                                                     3497
3498            beg_dt = expected.index[1]                               3498            beg_dt = expected.index[1]
3499            end_dt = expected.index[-2]                              3499            end_dt = expected.index[-2]
3500                                                                     3500
3501            # select w/iterator and where clause, single term, begin3501            # select w/iterator and where clause, single term, begin
                                                                of range                                                                of range
3502            where = "index >= '%s'" % beg_dt                         3502            where = "index >= '%s'" % beg_dt
3503            results = [s for s in store.select(                      3503            results = [s for s in store.select(
3504                'df', where=where, chunksize=chunksize)]             3504                'df', where=where, chunksize=chunksize)]
3505            result = concat(results)                                 3505            result = concat(results)
3506            rexpected = expected[expected.index >= beg_dt]           3506            rexpected = expected[expected.index >= beg_dt]
3507            tm.assert_frame_equal(rexpected, result)                 3507            tm.assert_frame_equal(rexpected, result)
3508                                                                     3508
3509            # select w/iterator and where clause, single term, end o3509            # select w/iterator and where clause, single term, end o
                                                                 f range                                                                 f range
3510            where = "index <= '%s'" % end_dt                         3510            where = "index <= '%s'" % end_dt
3511            results = [s for s in store.select(                      3511            results = [s for s in store.select(
3512                'df', where=where, chunksize=chunksize)]             3512                'df', where=where, chunksize=chunksize)]
3513            result = concat(results)                                 3513            result = concat(results)
3514            rexpected = expected[expected.index <= end_dt]           3514            rexpected = expected[expected.index <= end_dt]
3515            tm.assert_frame_equal(rexpected, result)                 3515            tm.assert_frame_equal(rexpected, result)
3516                                                                     3516
3517            # select w/iterator and where clause, inclusive range    3517            # select w/iterator and where clause, inclusive range
3518            where = "index >= '%s' & index <= '%s'" % (beg_dt, end_d3518            where = "index >= '%s' & index <= '%s'" % (beg_dt, end_d
                                                                      t)                                                                      t)
3519            results = [s for s in store.select(                      3519            results = [s for s in store.select(
3520                'df', where=where, chunksize=chunksize)]             3520                'df', where=where, chunksize=chunksize)]
3521            result = concat(results)                                 3521            result = concat(results)
3522            rexpected = expected[(expected.index >= beg_dt) &        3522            rexpected = expected[(expected.index >= beg_dt) &
3523                                 (expected.index <= end_dt)]         3523                                 (expected.index <= end_dt)]
3524            tm.assert_frame_equal(rexpected, result)                 3524            tm.assert_frame_equal(rexpected, result)
3525                                                                     3525
3526        # with iterator, empty where                                 3526        # with iterator, empty where
3527        with ensure_clean_store(self.path) as store:                 3527        with ensure_clean_store(self.path) as store:
3528                                                                     3528
3529            expected = tm.makeTimeDataFrame(100064, 'S')             3529            expected = tm.makeTimeDataFrame(100064, 'S')
3530            _maybe_remove(store, 'df')                               3530            _maybe_remove(store, 'df')
3531            store.append('df', expected)                             3531            store.append('df', expected)
3532                                                                     3532
3533            end_dt = expected.index[-1]                              3533            end_dt = expected.index[-1]
3534                                                                     3534
3535            # select w/iterator and where clause, single term, begin3535            # select w/iterator and where clause, single term, begin
                                                                of range                                                                of range
3536            where = "index > '%s'" % end_dt                          3536            where = "index > '%s'" % end_dt
3537            results = [s for s in store.select(                      3537            results = [s for s in store.select(
3538                'df', where=where, chunksize=chunksize)]             3538                'df', where=where, chunksize=chunksize)]
3539            assert 0 == len(results)                                 3539            assert 0 == len(results)
3540                                                                     3540
3541    def test_select_iterator_many_empty_frames(self):                3541    def test_select_iterator_many_empty_frames(self):
3542                                                                     3542
3543        # GH 8014                                                    3543        # GH 8014
3544        # using iterator and where clause can return many empty      3544        # using iterator and where clause can return many empty
3545        # frames.                                                    3545        # frames.
3546        chunksize = int(1e4)                                         3546        chunksize = int(1e4)
3547                                                                     3547
3548        # with iterator, range limited to the first chunk            3548        # with iterator, range limited to the first chunk
3549        with ensure_clean_store(self.path) as store:                 3549        with ensure_clean_store(self.path) as store:
3550                                                                     3550
3551            expected = tm.makeTimeDataFrame(100000, 'S')             3551            expected = tm.makeTimeDataFrame(100000, 'S')
3552            _maybe_remove(store, 'df')                               3552            _maybe_remove(store, 'df')
3553            store.append('df', expected)                             3553            store.append('df', expected)
3554                                                                     3554
3555            beg_dt = expected.index[0]                               3555            beg_dt = expected.index[0]
3556            end_dt = expected.index[chunksize - 1]                   3556            end_dt = expected.index[chunksize - 1]
3557                                                                     3557
3558            # select w/iterator and where clause, single term, begin3558            # select w/iterator and where clause, single term, begin
                                                                of range                                                                of range
3559            where = "index >= '%s'" % beg_dt                         3559            where = "index >= '%s'" % beg_dt
3560            results = [s for s in store.select(                      3560            results = [s for s in store.select(
3561                'df', where=where, chunksize=chunksize)]             3561                'df', where=where, chunksize=chunksize)]
3562            result = concat(results)                                 3562            result = concat(results)
3563            rexpected = expected[expected.index >= beg_dt]           3563            rexpected = expected[expected.index >= beg_dt]
3564            tm.assert_frame_equal(rexpected, result)                 3564            tm.assert_frame_equal(rexpected, result)
3565                                                                     3565
3566            # select w/iterator and where clause, single term, end o3566            # select w/iterator and where clause, single term, end o
                                                                 f range                                                                 f range
3567            where = "index <= '%s'" % end_dt                         3567            where = "index <= '%s'" % end_dt
3568            results = [s for s in store.select(                      3568            results = [s for s in store.select(
3569                'df', where=where, chunksize=chunksize)]             3569                'df', where=where, chunksize=chunksize)]
3570                                                                     3570
3571            assert len(results) == 1                                 3571            assert len(results) == 1
3572            result = concat(results)                                 3572            result = concat(results)
3573            rexpected = expected[expected.index <= end_dt]           3573            rexpected = expected[expected.index <= end_dt]
3574            tm.assert_frame_equal(rexpected, result)                 3574            tm.assert_frame_equal(rexpected, result)
3575                                                                     3575
3576            # select w/iterator and where clause, inclusive range    3576            # select w/iterator and where clause, inclusive range
3577            where = "index >= '%s' & index <= '%s'" % (beg_dt, end_d3577            where = "index >= '%s' & index <= '%s'" % (beg_dt, end_d
                                                                      t)                                                                      t)
3578            results = [s for s in store.select(                      3578            results = [s for s in store.select(
3579                'df', where=where, chunksize=chunksize)]             3579                'df', where=where, chunksize=chunksize)]
3580                                                                     3580
3581            # should be 1, is 10                                     3581            # should be 1, is 10
3582            assert len(results) == 1                                 3582            assert len(results) == 1
3583            result = concat(results)                                 3583            result = concat(results)
3584            rexpected = expected[(expected.index >= beg_dt) &        3584            rexpected = expected[(expected.index >= beg_dt) &
3585                                 (expected.index <= end_dt)]         3585                                 (expected.index <= end_dt)]
3586            tm.assert_frame_equal(rexpected, result)                 3586            tm.assert_frame_equal(rexpected, result)
3587                                                                     3587
3588            # select w/iterator and where clause which selects       3588            # select w/iterator and where clause which selects
3589            # *nothing*.                                             3589            # *nothing*.
3590            #                                                        3590            #
3591            # To be consistent with Python idiom I suggest this shou3591            # To be consistent with Python idiom I suggest this shou
                                                                      ld                                                                      ld
3592            # return [] e.g. `for e in []: print True` never prints  3592            # return [] e.g. `for e in []: print True` never prints
3593            # True.                                                  3593            # True.
3594                                                                     3594
3595            where = "index <= '%s' & index >= '%s'" % (beg_dt, end_d3595            where = "index <= '%s' & index >= '%s'" % (beg_dt, end_d
                                                                      t)                                                                      t)
3596            results = [s for s in store.select(                      3596            results = [s for s in store.select(
3597                'df', where=where, chunksize=chunksize)]             3597                'df', where=where, chunksize=chunksize)]
3598                                                                     3598
3599            # should be []                                           3599            # should be []
3600            assert len(results) == 0                                 3600            assert len(results) == 0
3601                                                                     3601
3602    def test_retain_index_attributes(self):                          3602    def test_retain_index_attributes(self):
3603                                                                     3603
3604        # GH 3499, losing frequency info on index recreation         3604        # GH 3499, losing frequency info on index recreation
3605        df = DataFrame(dict(                                         3605        df = DataFrame(dict(
3606            A=Series(lrange(3),                                      3606            A=Series(lrange(3),
3607                     index=date_range('2000-1-1', periods=3, freq='H3607                     index=date_range('2000-1-1', periods=3, freq='H
                                                                   '))))                                                                   '))))
3608                                                                     3608
3609        with ensure_clean_store(self.path) as store:                 3609        with ensure_clean_store(self.path) as store:
3610            _maybe_remove(store, 'data')                             3610            _maybe_remove(store, 'data')
3611            store.put('data', df, format='table')                    3611            store.put('data', df, format='table')
3612                                                                     3612
3613            result = store.get('data')                               3613            result = store.get('data')
3614            tm.assert_frame_equal(df, result)                        3614            tm.assert_frame_equal(df, result)
3615                                                                     3615
3616            for attr in ['freq', 'tz', 'name']:                      3616            for attr in ['freq', 'tz', 'name']:
3617                for idx in ['index', 'columns']:                     3617                for idx in ['index', 'columns']:
3618                    assert (getattr(getattr(df, idx), attr, None) == 3618                    assert (getattr(getattr(df, idx), attr, None) ==
3619                            getattr(getattr(result, idx), attr, None3619                            getattr(getattr(result, idx), attr, None
                                                                      ))                                                                      ))
3620                                                                     3620
3621            # try to append a table with a different frequency       3621            # try to append a table with a different frequency
3622            with catch_warnings(record=True):                        3622            with catch_warnings(record=True):
3623                df2 = DataFrame(dict(                                3623                df2 = DataFrame(dict(
3624                    A=Series(lrange(3),                              3624                    A=Series(lrange(3),
3625                             index=date_range('2002-1-1',            3625                             index=date_range('2002-1-1',
3626                                              periods=3, freq='D'))))3626                                              periods=3, freq='D'))))
3627                store.append('data', df2)                            3627                store.append('data', df2)
3628                                                                     3628
3629            assert store.get_storer('data').info['index']['freq'] is3629            assert store.get_storer('data').info['index']['freq'] is
                                                                    None                                                                    None
3630                                                                     3630
3631            # this is ok                                             3631            # this is ok
3632            _maybe_remove(store, 'df2')                              3632            _maybe_remove(store, 'df2')
3633            df2 = DataFrame(dict(                                    3633            df2 = DataFrame(dict(
3634                A=Series(lrange(3),                                  3634                A=Series(lrange(3),
3635                         index=[Timestamp('20010101'), Timestamp('203635                         index=[Timestamp('20010101'), Timestamp('20
                                                               010102'),                                                               010102'),
3636                                Timestamp('20020101')])))            3636                                Timestamp('20020101')])))
3637            store.append('df2', df2)                                 3637            store.append('df2', df2)
3638            df3 = DataFrame(dict(                                    3638            df3 = DataFrame(dict(
3639                A=Series(lrange(3),                                  3639                A=Series(lrange(3),
3640                         index=date_range('2002-1-1', periods=3,     3640                         index=date_range('2002-1-1', periods=3,
3641                                          freq='D'))))               3641                                          freq='D'))))
3642            store.append('df2', df3)                                 3642            store.append('df2', df3)
3643                                                                     3643
3644    def test_retain_index_attributes2(self):                         3644    def test_retain_index_attributes2(self):
3645        with ensure_clean_path(self.path) as path:                   3645        with ensure_clean_path(self.path) as path:
3646                                                                     3646
3647            with catch_warnings(record=True):                        3647            with catch_warnings(record=True):
3648                                                                     3648
3649                df = DataFrame(dict(                                 3649                df = DataFrame(dict(
3650                    A=Series(lrange(3),                              3650                    A=Series(lrange(3),
3651                             index=date_range('2000-1-1',            3651                             index=date_range('2000-1-1',
3652                                              periods=3, freq='H'))))3652                                              periods=3, freq='H'))))
3653                df.to_hdf(path, 'data', mode='w', append=True)       3653                df.to_hdf(path, 'data', mode='w', append=True)
3654                df2 = DataFrame(dict(                                3654                df2 = DataFrame(dict(
3655                    A=Series(lrange(3),                              3655                    A=Series(lrange(3),
3656                             index=date_range('2002-1-1', periods=3, 3656                             index=date_range('2002-1-1', periods=3,
3657                                              freq='D'))))           3657                                              freq='D'))))
3658                df2.to_hdf(path, 'data', append=True)                3658                df2.to_hdf(path, 'data', append=True)
3659                                                                     3659
3660                idx = date_range('2000-1-1', periods=3, freq='H')    3660                idx = date_range('2000-1-1', periods=3, freq='H')
3661                idx.name = 'foo'                                     3661                idx.name = 'foo'
3662                df = DataFrame(dict(A=Series(lrange(3), index=idx))) 3662                df = DataFrame(dict(A=Series(lrange(3), index=idx)))
3663                df.to_hdf(path, 'data', mode='w', append=True)       3663                df.to_hdf(path, 'data', mode='w', append=True)
3664                                                                     3664
3665            assert read_hdf(path, 'data').index.name == 'foo'        3665            assert read_hdf(path, 'data').index.name == 'foo'
3666                                                                     3666
3667            with catch_warnings(record=True):                        3667            with catch_warnings(record=True):
3668                                                                     3668
3669                idx2 = date_range('2001-1-1', periods=3, freq='H')   3669                idx2 = date_range('2001-1-1', periods=3, freq='H')
3670                idx2.name = 'bar'                                    3670                idx2.name = 'bar'
3671                df2 = DataFrame(dict(A=Series(lrange(3), index=idx2)3671                df2 = DataFrame(dict(A=Series(lrange(3), index=idx2)
                                                                      ))                                                                      ))
3672                df2.to_hdf(path, 'data', append=True)                3672                df2.to_hdf(path, 'data', append=True)
3673                                                                     3673
3674            assert read_hdf(path, 'data').index.name is None         3674            assert read_hdf(path, 'data').index.name is None
3675                                                                     3675
3676    def test_panel_select(self):                                     3676    def test_panel_select(self):
3677                                                                     3677
3678        with ensure_clean_store(self.path) as store:                 3678        with ensure_clean_store(self.path) as store:
3679                                                                     3679
3680            with catch_warnings(record=True):                        3680            with catch_warnings(record=True):
3681                                                                     3681
3682                wp = tm.makePanel()                                  3682                wp = tm.makePanel()
3683                                                                     3683
3684                store.put('wp', wp, format='table')                  3684                store.put('wp', wp, format='table')
3685                date = wp.major_axis[len(wp.major_axis) // 2]        3685                date = wp.major_axis[len(wp.major_axis) // 2]
3686                                                                     3686
3687                crit1 = ('major_axis>=date')                         3687                crit1 = ('major_axis>=date')
3688                crit2 = ("minor_axis=['A', 'D']")                    3688                crit2 = ("minor_axis=['A', 'D']")
3689                                                                     3689
3690                result = store.select('wp', [crit1, crit2])          3690                result = store.select('wp', [crit1, crit2])
3691                expected = wp.truncate(before=date).reindex(minor=['3691                expected = wp.truncate(before=date).reindex(minor=['
                                                               A', 'D'])                                                               A', 'D'])
3692                assert_panel_equal(result, expected)                 3692                assert_panel_equal(result, expected)
3693                                                                     3693
3694                result = store.select(                               3694                result = store.select(
3695                    'wp', ['major_axis>="20000124"',                 3695                    'wp', ['major_axis>="20000124"',
3696                           ("minor_axis=['A', 'B']")])               3696                           ("minor_axis=['A', 'B']")])
3697                expected = wp.truncate(                              3697                expected = wp.truncate(
3698                    before='20000124').reindex(minor=['A', 'B'])     3698                    before='20000124').reindex(minor=['A', 'B'])
3699                assert_panel_equal(result, expected)                 3699                assert_panel_equal(result, expected)
3700                                                                     3700
3701    def test_frame_select(self):                                     3701    def test_frame_select(self):
3702                                                                     3702
3703        df = tm.makeTimeDataFrame()                                  3703        df = tm.makeTimeDataFrame()
3704                                                                     3704
3705        with ensure_clean_store(self.path) as store:                 3705        with ensure_clean_store(self.path) as store:
3706            store.put('frame', df, format='table')                   3706            store.put('frame', df, format='table')
3707            date = df.index[len(df) // 2]                            3707            date = df.index[len(df) // 2]
3708                                                                     3708
3709            crit1 = Term('index>=date')                              3709            crit1 = Term('index>=date')
3710            assert crit1.env.scope['date'] == date                   3710            assert crit1.env.scope['date'] == date
3711                                                                     3711
3712            crit2 = ("columns=['A', 'D']")                           3712            crit2 = ("columns=['A', 'D']")
3713            crit3 = ('columns=A')                                    3713            crit3 = ('columns=A')
3714                                                                     3714
3715            result = store.select('frame', [crit1, crit2])           3715            result = store.select('frame', [crit1, crit2])
3716            expected = df.loc[date:, ['A', 'D']]                     3716            expected = df.loc[date:, ['A', 'D']]
3717            tm.assert_frame_equal(result, expected)                  3717            tm.assert_frame_equal(result, expected)
3718                                                                     3718
3719            result = store.select('frame', [crit3])                  3719            result = store.select('frame', [crit3])
3720            expected = df.loc[:, ['A']]                              3720            expected = df.loc[:, ['A']]
3721            tm.assert_frame_equal(result, expected)                  3721            tm.assert_frame_equal(result, expected)
3722                                                                     3722
3723            # invalid terms                                          3723            # invalid terms
3724            df = tm.makeTimeDataFrame()                              3724            df = tm.makeTimeDataFrame()
3725            store.append('df_time', df)                              3725            store.append('df_time', df)
3726            pytest.raises(                                           3726            pytest.raises(
3727                ValueError, store.select, 'df_time', "index>0")      3727                ValueError, store.select, 'df_time', "index>0")
3728                                                                     3728
3729            # can't select if not written as table                   3729            # can't select if not written as table
3730            # store['frame'] = df                                    3730            # store['frame'] = df
3731            # pytest.raises(ValueError, store.select,                3731            # pytest.raises(ValueError, store.select,
3732            #                  'frame', [crit1, crit2])              3732            #                  'frame', [crit1, crit2])
3733                                                                     3733
3734    def test_frame_select_complex(self):                             3734    def test_frame_select_complex(self):
3735        # select via complex criteria                                3735        # select via complex criteria
3736                                                                     3736
3737        df = tm.makeTimeDataFrame()                                  3737        df = tm.makeTimeDataFrame()
3738        df['string'] = 'foo'                                         3738        df['string'] = 'foo'
3739        df.loc[df.index[0:4], 'string'] = 'bar'                      3739        df.loc[df.index[0:4], 'string'] = 'bar'
3740                                                                     3740
3741        with ensure_clean_store(self.path) as store:                 3741        with ensure_clean_store(self.path) as store:
3742            store.put('df', df, format='table', data_columns=['strin3742            store.put('df', df, format='table', data_columns=['strin
                                                                    g'])                                                                    g'])
3743                                                                     3743
3744            # empty                                                  3744            # empty
3745            result = store.select('df', 'index>df.index[3] & string=3745            result = store.select('df', 'index>df.index[3] & string=
                                                                 "bar"')                                                                 "bar"')
3746            expected = df.loc[(df.index > df.index[3]) & (df.string 3746            expected = df.loc[(df.index > df.index[3]) & (df.string 
                                                              == 'bar')]                                                              == 'bar')]
3747            tm.assert_frame_equal(result, expected)                  3747            tm.assert_frame_equal(result, expected)
3748                                                                     3748
3749            result = store.select('df', 'index>df.index[3] & string=3749            result = store.select('df', 'index>df.index[3] & string=
                                                                 "foo"')                                                                 "foo"')
3750            expected = df.loc[(df.index > df.index[3]) & (df.string 3750            expected = df.loc[(df.index > df.index[3]) & (df.string 
                                                              == 'foo')]                                                              == 'foo')]
3751            tm.assert_frame_equal(result, expected)                  3751            tm.assert_frame_equal(result, expected)
3752                                                                     3752
3753            # or                                                     3753            # or
3754            result = store.select('df', 'index>df.index[3] | string=3754            result = store.select('df', 'index>df.index[3] | string=
                                                                 "bar"')                                                                 "bar"')
3755            expected = df.loc[(df.index > df.index[3]) | (df.string 3755            expected = df.loc[(df.index > df.index[3]) | (df.string 
                                                              == 'bar')]                                                              == 'bar')]
3756            tm.assert_frame_equal(result, expected)                  3756            tm.assert_frame_equal(result, expected)
3757                                                                     3757
3758            result = store.select('df', '(index>df.index[3] & '      3758            result = store.select('df', '(index>df.index[3] & '
3759                                  'index<=df.index[6]) | string="bar3759                                  'index<=df.index[6]) | string="bar
                                                                     "')                                                                     "')
3760            expected = df.loc[((df.index > df.index[3]) & (          3760            expected = df.loc[((df.index > df.index[3]) & (
3761                df.index <= df.index[6])) | (df.string == 'bar')]    3761                df.index <= df.index[6])) | (df.string == 'bar')]
3762            tm.assert_frame_equal(result, expected)                  3762            tm.assert_frame_equal(result, expected)
3763                                                                     3763
3764            # invert                                                 3764            # invert
3765            result = store.select('df', 'string!="bar"')             3765            result = store.select('df', 'string!="bar"')
3766            expected = df.loc[df.string != 'bar']                    3766            expected = df.loc[df.string != 'bar']
3767            tm.assert_frame_equal(result, expected)                  3767            tm.assert_frame_equal(result, expected)
3768                                                                     3768
3769            # invert not implemented in numexpr :(                   3769            # invert not implemented in numexpr :(
3770            pytest.raises(NotImplementedError,                       3770            pytest.raises(NotImplementedError,
3771                          store.select, 'df', '~(string="bar")')     3771                          store.select, 'df', '~(string="bar")')
3772                                                                     3772
3773            # invert ok for filters                                  3773            # invert ok for filters
3774            result = store.select('df', "~(columns=['A','B'])")      3774            result = store.select('df', "~(columns=['A','B'])")
3775            expected = df.loc[:, df.columns.difference(['A', 'B'])]  3775            expected = df.loc[:, df.columns.difference(['A', 'B'])]
3776            tm.assert_frame_equal(result, expected)                  3776            tm.assert_frame_equal(result, expected)
3777                                                                     3777
3778            # in                                                     3778            # in
3779            result = store.select(                                   3779            result = store.select(
3780                'df', "index>df.index[3] & columns in ['A','B']")    3780                'df', "index>df.index[3] & columns in ['A','B']")
3781            expected = df.loc[df.index > df.index[3]].reindex(column3781            expected = df.loc[df.index > df.index[3]].reindex(column
                                                                     s=[                                                                     s=[
3782                                                              'A', '3782                                                              'A', '
                                                                    B'])                                                                    B'])
3783            tm.assert_frame_equal(result, expected)                  3783            tm.assert_frame_equal(result, expected)
3784                                                                     3784
3785    def test_frame_select_complex2(self):                            3785    def test_frame_select_complex2(self):
3786                                                                     3786
3787        with ensure_clean_path(['parms.hdf', 'hist.hdf']) as paths:  3787        with ensure_clean_path(['parms.hdf', 'hist.hdf']) as paths:
3788                                                                     3788
3789            pp, hh = paths                                           3789            pp, hh = paths
3790                                                                     3790
3791            # use non-trivial selection criteria                     3791            # use non-trivial selection criteria
3792            parms = DataFrame({'A': [1, 1, 2, 2, 3]})                3792            parms = DataFrame({'A': [1, 1, 2, 2, 3]})
3793            parms.to_hdf(pp, 'df', mode='w',                         3793            parms.to_hdf(pp, 'df', mode='w',
3794                         format='table', data_columns=['A'])         3794                         format='table', data_columns=['A'])
3795                                                                     3795
3796            selection = read_hdf(pp, 'df', where='A=[2,3]')          3796            selection = read_hdf(pp, 'df', where='A=[2,3]')
3797            hist = DataFrame(np.random.randn(25, 1),                 3797            hist = DataFrame(np.random.randn(25, 1),
3798                             columns=['data'],                       3798                             columns=['data'],
3799                             index=MultiIndex.from_tuples(           3799                             index=MultiIndex.from_tuples(
3800                                 [(i, j) for i in range(5)           3800                                 [(i, j) for i in range(5)
3801                                  for j in range(5)],                3801                                  for j in range(5)],
3802                                 names=['l1', 'l2']))                3802                                 names=['l1', 'l2']))
3803                                                                     3803
3804            hist.to_hdf(hh, 'df', mode='w', format='table')          3804            hist.to_hdf(hh, 'df', mode='w', format='table')
3805                                                                     3805
3806            expected = read_hdf(hh, 'df', where='l1=[2, 3, 4]')      3806            expected = read_hdf(hh, 'df', where='l1=[2, 3, 4]')
3807                                                                     3807
3808            # sccope with list like                                  3808            # sccope with list like
3809            l = selection.index.tolist()  # noqa                     3809            l = selection.index.tolist()  # noqa
3810            store = HDFStore(hh)                                     3810            store = HDFStore(hh)
3811            result = store.select('df', where='l1=l')                3811            result = store.select('df', where='l1=l')
3812            assert_frame_equal(result, expected)                     3812            assert_frame_equal(result, expected)
3813            store.close()                                            3813            store.close()
3814                                                                     3814
3815            result = read_hdf(hh, 'df', where='l1=l')                3815            result = read_hdf(hh, 'df', where='l1=l')
3816            assert_frame_equal(result, expected)                     3816            assert_frame_equal(result, expected)
3817                                                                     3817
3818            # index                                                  3818            # index
3819            index = selection.index  # noqa                          3819            index = selection.index  # noqa
3820            result = read_hdf(hh, 'df', where='l1=index')            3820            result = read_hdf(hh, 'df', where='l1=index')
3821            assert_frame_equal(result, expected)                     3821            assert_frame_equal(result, expected)
3822                                                                     3822
3823            result = read_hdf(hh, 'df', where='l1=selection.index')  3823            result = read_hdf(hh, 'df', where='l1=selection.index')
3824            assert_frame_equal(result, expected)                     3824            assert_frame_equal(result, expected)
3825                                                                     3825
3826            result = read_hdf(hh, 'df', where='l1=selection.index.to3826            result = read_hdf(hh, 'df', where='l1=selection.index.to
                                                                list()')                                                                list()')
3827            assert_frame_equal(result, expected)                     3827            assert_frame_equal(result, expected)
3828                                                                     3828
3829            result = read_hdf(hh, 'df', where='l1=list(selection.ind3829            result = read_hdf(hh, 'df', where='l1=list(selection.ind
                                                                   ex)')                                                                   ex)')
3830            assert_frame_equal(result, expected)                     3830            assert_frame_equal(result, expected)
3831                                                                     3831
3832            # sccope with index                                      3832            # sccope with index
3833            store = HDFStore(hh)                                     3833            store = HDFStore(hh)
3834                                                                     3834
3835            result = store.select('df', where='l1=index')            3835            result = store.select('df', where='l1=index')
3836            assert_frame_equal(result, expected)                     3836            assert_frame_equal(result, expected)
3837                                                                     3837
3838            result = store.select('df', where='l1=selection.index')  3838            result = store.select('df', where='l1=selection.index')
3839            assert_frame_equal(result, expected)                     3839            assert_frame_equal(result, expected)
3840                                                                     3840
3841            result = store.select('df', where='l1=selection.index.to3841            result = store.select('df', where='l1=selection.index.to
                                                                list()')                                                                list()')
3842            assert_frame_equal(result, expected)                     3842            assert_frame_equal(result, expected)
3843                                                                     3843
3844            result = store.select('df', where='l1=list(selection.ind3844            result = store.select('df', where='l1=list(selection.ind
                                                                   ex)')                                                                   ex)')
3845            assert_frame_equal(result, expected)                     3845            assert_frame_equal(result, expected)
3846                                                                     3846
3847            store.close()                                            3847            store.close()
3848                                                                     3848
3849    def test_invalid_filtering(self):                                3849    def test_invalid_filtering(self):
3850                                                                     3850
3851        # can't use more than one filter (atm)                       3851        # can't use more than one filter (atm)
3852                                                                     3852
3853        df = tm.makeTimeDataFrame()                                  3853        df = tm.makeTimeDataFrame()
3854                                                                     3854
3855        with ensure_clean_store(self.path) as store:                 3855        with ensure_clean_store(self.path) as store:
3856            store.put('df', df, format='table')                      3856            store.put('df', df, format='table')
3857                                                                     3857
3858            # not implemented                                        3858            # not implemented
3859            pytest.raises(NotImplementedError, store.select,         3859            pytest.raises(NotImplementedError, store.select,
3860                          'df', "columns=['A'] | columns=['B']")     3860                          'df', "columns=['A'] | columns=['B']")
3861                                                                     3861
3862            # in theory we could deal with this                      3862            # in theory we could deal with this
3863            pytest.raises(NotImplementedError, store.select,         3863            pytest.raises(NotImplementedError, store.select,
3864                          'df', "columns=['A','B'] & columns=['C']") 3864                          'df', "columns=['A','B'] & columns=['C']")
3865                                                                     3865
3866    def test_string_select(self):                                    3866    def test_string_select(self):
3867        # GH 2973                                                    3867        # GH 2973
3868        with ensure_clean_store(self.path) as store:                 3868        with ensure_clean_store(self.path) as store:
3869                                                                     3869
3870            df = tm.makeTimeDataFrame()                              3870            df = tm.makeTimeDataFrame()
3871                                                                     3871
3872            # test string ==/!=                                      3872            # test string ==/!=
3873            df['x'] = 'none'                                         3873            df['x'] = 'none'
3874            df.loc[2:7, 'x'] = ''                                    3874            df.loc[2:7, 'x'] = ''
3875                                                                     3875
3876            store.append('df', df, data_columns=['x'])               3876            store.append('df', df, data_columns=['x'])
3877                                                                     3877
3878            result = store.select('df', 'x=none')                    3878            result = store.select('df', 'x=none')
3879            expected = df[df.x == 'none']                            3879            expected = df[df.x == 'none']
3880            assert_frame_equal(result, expected)                     3880            assert_frame_equal(result, expected)
3881                                                                     3881
3882            try:                                                     3882            try:
3883                result = store.select('df', 'x!=none')               3883                result = store.select('df', 'x!=none')
3884                expected = df[df.x != 'none']                        3884                expected = df[df.x != 'none']
3885                assert_frame_equal(result, expected)                 3885                assert_frame_equal(result, expected)
3886            except Exception as detail:                              3886            except Exception as detail:
3887                pprint_thing("[{0}]".format(detail))                 3887                pprint_thing("[{0}]".format(detail))
3888                pprint_thing(store)                                  3888                pprint_thing(store)
3889                pprint_thing(expected)                               3889                pprint_thing(expected)
3890                                                                     3890
3891            df2 = df.copy()                                          3891            df2 = df.copy()
3892            df2.loc[df2.x == '', 'x'] = np.nan                       3892            df2.loc[df2.x == '', 'x'] = np.nan
3893                                                                     3893
3894            store.append('df2', df2, data_columns=['x'])             3894            store.append('df2', df2, data_columns=['x'])
3895            result = store.select('df2', 'x!=none')                  3895            result = store.select('df2', 'x!=none')
3896            expected = df2[isnull(df2.x)]                            3896            expected = df2[isnull(df2.x)]
3897            assert_frame_equal(result, expected)                     3897            assert_frame_equal(result, expected)
3898                                                                     3898
3899            # int ==/!=                                              3899            # int ==/!=
3900            df['int'] = 1                                            3900            df['int'] = 1
3901            df.loc[2:7, 'int'] = 2                                   3901            df.loc[2:7, 'int'] = 2
3902                                                                     3902
3903            store.append('df3', df, data_columns=['int'])            3903            store.append('df3', df, data_columns=['int'])
3904                                                                     3904
3905            result = store.select('df3', 'int=2')                    3905            result = store.select('df3', 'int=2')
3906            expected = df[df.int == 2]                               3906            expected = df[df.int == 2]
3907            assert_frame_equal(result, expected)                     3907            assert_frame_equal(result, expected)
3908                                                                     3908
3909            result = store.select('df3', 'int!=2')                   3909            result = store.select('df3', 'int!=2')
3910            expected = df[df.int != 2]                               3910            expected = df[df.int != 2]
3911            assert_frame_equal(result, expected)                     3911            assert_frame_equal(result, expected)
3912                                                                     3912
3913    def test_read_column(self):                                      3913    def test_read_column(self):
3914                                                                     3914
3915        df = tm.makeTimeDataFrame()                                  3915        df = tm.makeTimeDataFrame()
3916                                                                     3916
3917        with ensure_clean_store(self.path) as store:                 3917        with ensure_clean_store(self.path) as store:
3918            _maybe_remove(store, 'df')                               3918            _maybe_remove(store, 'df')
3919            store.append('df', df)                                   3919            store.append('df', df)
3920                                                                     3920
3921            # error                                                  3921            # error
3922            pytest.raises(KeyError, store.select_column, 'df', 'foo')3922            pytest.raises(KeyError, store.select_column, 'df', 'foo')
3923                                                                     3923
3924            def f():                                                 3924            def f():
3925                store.select_column('df', 'index', where=['index>5'])3925                store.select_column('df', 'index', where=['index>5'])
3926            pytest.raises(Exception, f)                              3926            pytest.raises(Exception, f)
3927                                                                     3927
3928            # valid                                                  3928            # valid
3929            result = store.select_column('df', 'index')              3929            result = store.select_column('df', 'index')
3930            tm.assert_almost_equal(result.values, Series(df.index).v3930            tm.assert_almost_equal(result.values, Series(df.index).v
                                                                  alues)                                                                  alues)
3931            assert isinstance(result, Series)                        3931            assert isinstance(result, Series)
3932                                                                     3932
3933            # not a data indexable column                            3933            # not a data indexable column
3934            pytest.raises(                                           3934            pytest.raises(
3935                ValueError, store.select_column, 'df', 'values_block3935                ValueError, store.select_column, 'df', 'values_block
                                                                    _0')                                                                    _0')
3936                                                                     3936
3937            # a data column                                          3937            # a data column
3938            df2 = df.copy()                                          3938            df2 = df.copy()
3939            df2['string'] = 'foo'                                    3939            df2['string'] = 'foo'
3940            store.append('df2', df2, data_columns=['string'])        3940            store.append('df2', df2, data_columns=['string'])
3941            result = store.select_column('df2', 'string')            3941            result = store.select_column('df2', 'string')
3942            tm.assert_almost_equal(result.values, df2['string'].valu3942            tm.assert_almost_equal(result.values, df2['string'].valu
                                                                     es)                                                                     es)
3943                                                                     3943
3944            # a data column with NaNs, result excludes the NaNs      3944            # a data column with NaNs, result excludes the NaNs
3945            df3 = df.copy()                                          3945            df3 = df.copy()
3946            df3['string'] = 'foo'                                    3946            df3['string'] = 'foo'
3947            df3.loc[4:6, 'string'] = np.nan                          3947            df3.loc[4:6, 'string'] = np.nan
3948            store.append('df3', df3, data_columns=['string'])        3948            store.append('df3', df3, data_columns=['string'])
3949            result = store.select_column('df3', 'string')            3949            result = store.select_column('df3', 'string')
3950            tm.assert_almost_equal(result.values, df3['string'].valu3950            tm.assert_almost_equal(result.values, df3['string'].valu
                                                                     es)                                                                     es)
3951                                                                     3951
3952            # start/stop                                             3952            # start/stop
3953            result = store.select_column('df3', 'string', start=2)   3953            result = store.select_column('df3', 'string', start=2)
3954            tm.assert_almost_equal(result.values, df3['string'].valu3954            tm.assert_almost_equal(result.values, df3['string'].valu
                                                                 es[2:])                                                                 es[2:])
3955                                                                     3955
3956            result = store.select_column('df3', 'string', start=-2)  3956            result = store.select_column('df3', 'string', start=-2)
3957            tm.assert_almost_equal(result.values, df3['string'].valu3957            tm.assert_almost_equal(result.values, df3['string'].valu
                                                                es[-2:])                                                                es[-2:])
3958                                                                     3958
3959            result = store.select_column('df3', 'string', stop=2)    3959            result = store.select_column('df3', 'string', stop=2)
3960            tm.assert_almost_equal(result.values, df3['string'].valu3960            tm.assert_almost_equal(result.values, df3['string'].valu
                                                                 es[:2])                                                                 es[:2])
3961                                                                     3961
3962            result = store.select_column('df3', 'string', stop=-2)   3962            result = store.select_column('df3', 'string', stop=-2)
3963            tm.assert_almost_equal(result.values, df3['string'].valu3963            tm.assert_almost_equal(result.values, df3['string'].valu
                                                                es[:-2])                                                                es[:-2])
3964                                                                     3964
3965            result = store.select_column('df3', 'string', start=2, s3965            result = store.select_column('df3', 'string', start=2, s
                                                                 top=-2)                                                                 top=-2)
3966            tm.assert_almost_equal(result.values, df3['string'].valu3966            tm.assert_almost_equal(result.values, df3['string'].valu
                                                               es[2:-2])                                                               es[2:-2])
3967                                                                     3967
3968            result = store.select_column('df3', 'string', start=-2, 3968            result = store.select_column('df3', 'string', start=-2, 
                                                                 stop=2)                                                                 stop=2)
3969            tm.assert_almost_equal(result.values, df3['string'].valu3969            tm.assert_almost_equal(result.values, df3['string'].valu
                                                               es[-2:2])                                                               es[-2:2])
3970                                                                     3970
3971            # GH 10392 - make sure column name is preserved          3971            # GH 10392 - make sure column name is preserved
3972            df4 = DataFrame({'A': np.random.randn(10), 'B': 'foo'})  3972            df4 = DataFrame({'A': np.random.randn(10), 'B': 'foo'})
3973            store.append('df4', df4, data_columns=True)              3973            store.append('df4', df4, data_columns=True)
3974            expected = df4['B']                                      3974            expected = df4['B']
3975            result = store.select_column('df4', 'B')                 3975            result = store.select_column('df4', 'B')
3976            tm.assert_series_equal(result, expected)                 3976            tm.assert_series_equal(result, expected)
3977                                                                     3977
3978    def test_coordinates(self):                                      3978    def test_coordinates(self):
3979        df = tm.makeTimeDataFrame()                                  3979        df = tm.makeTimeDataFrame()
3980                                                                     3980
3981        with ensure_clean_store(self.path) as store:                 3981        with ensure_clean_store(self.path) as store:
3982                                                                     3982
3983            _maybe_remove(store, 'df')                               3983            _maybe_remove(store, 'df')
3984            store.append('df', df)                                   3984            store.append('df', df)
3985                                                                     3985
3986            # all                                                    3986            # all
3987            c = store.select_as_coordinates('df')                    3987            c = store.select_as_coordinates('df')
3988            assert((c.values == np.arange(len(df.index))).all())     3988            assert((c.values == np.arange(len(df.index))).all())
3989                                                                     3989
3990            # get coordinates back & test vs frame                   3990            # get coordinates back & test vs frame
3991            _maybe_remove(store, 'df')                               3991            _maybe_remove(store, 'df')
3992                                                                     3992
3993            df = DataFrame(dict(A=lrange(5), B=lrange(5)))           3993            df = DataFrame(dict(A=lrange(5), B=lrange(5)))
3994            store.append('df', df)                                   3994            store.append('df', df)
3995            c = store.select_as_coordinates('df', ['index<3'])       3995            c = store.select_as_coordinates('df', ['index<3'])
3996            assert((c.values == np.arange(3)).all())                 3996            assert((c.values == np.arange(3)).all())
3997            result = store.select('df', where=c)                     3997            result = store.select('df', where=c)
3998            expected = df.loc[0:2, :]                                3998            expected = df.loc[0:2, :]
3999            tm.assert_frame_equal(result, expected)                  3999            tm.assert_frame_equal(result, expected)
4000                                                                     4000
4001            c = store.select_as_coordinates('df', ['index>=3', 'inde4001            c = store.select_as_coordinates('df', ['index>=3', 'inde
                                                                 x<=4'])                                                                 x<=4'])
4002            assert((c.values == np.arange(2) + 3).all())             4002            assert((c.values == np.arange(2) + 3).all())
4003            result = store.select('df', where=c)                     4003            result = store.select('df', where=c)
4004            expected = df.loc[3:4, :]                                4004            expected = df.loc[3:4, :]
4005            tm.assert_frame_equal(result, expected)                  4005            tm.assert_frame_equal(result, expected)
4006            assert isinstance(c, Index)                              4006            assert isinstance(c, Index)
4007                                                                     4007
4008            # multiple tables                                        4008            # multiple tables
4009            _maybe_remove(store, 'df1')                              4009            _maybe_remove(store, 'df1')
4010            _maybe_remove(store, 'df2')                              4010            _maybe_remove(store, 'df2')
4011            df1 = tm.makeTimeDataFrame()                             4011            df1 = tm.makeTimeDataFrame()
4012            df2 = tm.makeTimeDataFrame().rename(columns=lambda x: "%4012            df2 = tm.makeTimeDataFrame().rename(columns=lambda x: "%
                                                               s_2" % x)                                                               s_2" % x)
4013            store.append('df1', df1, data_columns=['A', 'B'])        4013            store.append('df1', df1, data_columns=['A', 'B'])
4014            store.append('df2', df2)                                 4014            store.append('df2', df2)
4015                                                                     4015
4016            c = store.select_as_coordinates('df1', ['A>0', 'B>0'])   4016            c = store.select_as_coordinates('df1', ['A>0', 'B>0'])
4017            df1_result = store.select('df1', c)                      4017            df1_result = store.select('df1', c)
4018            df2_result = store.select('df2', c)                      4018            df2_result = store.select('df2', c)
4019            result = concat([df1_result, df2_result], axis=1)        4019            result = concat([df1_result, df2_result], axis=1)
4020                                                                     4020
4021            expected = concat([df1, df2], axis=1)                    4021            expected = concat([df1, df2], axis=1)
4022            expected = expected[(expected.A > 0) & (expected.B > 0)] 4022            expected = expected[(expected.A > 0) & (expected.B > 0)]
4023            tm.assert_frame_equal(result, expected)                  4023            tm.assert_frame_equal(result, expected)
4024                                                                     4024
4025        # pass array/mask as the coordinates                         4025        # pass array/mask as the coordinates
4026        with ensure_clean_store(self.path) as store:                 4026        with ensure_clean_store(self.path) as store:
4027                                                                     4027
4028            df = DataFrame(np.random.randn(1000, 2),                 4028            df = DataFrame(np.random.randn(1000, 2),
4029                           index=date_range('20000101', periods=10004029                           index=date_range('20000101', periods=1000
                                                                      ))                                                                      ))
4030            store.append('df', df)                                   4030            store.append('df', df)
4031            c = store.select_column('df', 'index')                   4031            c = store.select_column('df', 'index')
4032            where = c[DatetimeIndex(c).month == 5].index             4032            where = c[DatetimeIndex(c).month == 5].index
4033            expected = df.iloc[where]                                4033            expected = df.iloc[where]
4034                                                                     4034
4035            # locations                                              4035            # locations
4036            result = store.select('df', where=where)                 4036            result = store.select('df', where=where)
4037            tm.assert_frame_equal(result, expected)                  4037            tm.assert_frame_equal(result, expected)
4038                                                                     4038
4039            # boolean                                                4039            # boolean
4040            result = store.select('df', where=where)                 4040            result = store.select('df', where=where)
4041            tm.assert_frame_equal(result, expected)                  4041            tm.assert_frame_equal(result, expected)
4042                                                                     4042
4043            # invalid                                                4043            # invalid
4044            pytest.raises(ValueError, store.select, 'df',            4044            pytest.raises(ValueError, store.select, 'df',
4045                          where=np.arange(len(df), dtype='float64')) 4045                          where=np.arange(len(df), dtype='float64'))
4046            pytest.raises(ValueError, store.select, 'df',            4046            pytest.raises(ValueError, store.select, 'df',
4047                          where=np.arange(len(df) + 1))              4047                          where=np.arange(len(df) + 1))
4048            pytest.raises(ValueError, store.select, 'df',            4048            pytest.raises(ValueError, store.select, 'df',
4049                          where=np.arange(len(df)), start=5)         4049                          where=np.arange(len(df)), start=5)
4050            pytest.raises(ValueError, store.select, 'df',            4050            pytest.raises(ValueError, store.select, 'df',
4051                          where=np.arange(len(df)), start=5, stop=10)4051                          where=np.arange(len(df)), start=5, stop=10)
4052                                                                     4052
4053            # selection with filter                                  4053            # selection with filter
4054            selection = date_range('20000101', periods=500)          4054            selection = date_range('20000101', periods=500)
4055            result = store.select('df', where='index in selection')  4055            result = store.select('df', where='index in selection')
4056            expected = df[df.index.isin(selection)]                  4056            expected = df[df.index.isin(selection)]
4057            tm.assert_frame_equal(result, expected)                  4057            tm.assert_frame_equal(result, expected)
4058                                                                     4058
4059            # list                                                   4059            # list
4060            df = DataFrame(np.random.randn(10, 2))                   4060            df = DataFrame(np.random.randn(10, 2))
4061            store.append('df2', df)                                  4061            store.append('df2', df)
4062            result = store.select('df2', where=[0, 3, 5])            4062            result = store.select('df2', where=[0, 3, 5])
4063            expected = df.iloc[[0, 3, 5]]                            4063            expected = df.iloc[[0, 3, 5]]
4064            tm.assert_frame_equal(result, expected)                  4064            tm.assert_frame_equal(result, expected)
4065                                                                     4065
4066            # boolean                                                4066            # boolean
4067            where = [True] * 10                                      4067            where = [True] * 10
4068            where[-2] = False                                        4068            where[-2] = False
4069            result = store.select('df2', where=where)                4069            result = store.select('df2', where=where)
4070            expected = df.loc[where]                                 4070            expected = df.loc[where]
4071            tm.assert_frame_equal(result, expected)                  4071            tm.assert_frame_equal(result, expected)
4072                                                                     4072
4073            # start/stop                                             4073            # start/stop
4074            result = store.select('df2', start=5, stop=10)           4074            result = store.select('df2', start=5, stop=10)
4075            expected = df[5:10]                                      4075            expected = df[5:10]
4076            tm.assert_frame_equal(result, expected)                  4076            tm.assert_frame_equal(result, expected)
4077                                                                     4077
4078    def test_append_to_multiple(self):                               4078    def test_append_to_multiple(self):
4079        df1 = tm.makeTimeDataFrame()                                 4079        df1 = tm.makeTimeDataFrame()
4080        df2 = tm.makeTimeDataFrame().rename(columns=lambda x: "%s_2"4080        df2 = tm.makeTimeDataFrame().rename(columns=lambda x: "%s_2"
                                                                    % x)                                                                    % x)
4081        df2['foo'] = 'bar'                                           4081        df2['foo'] = 'bar'
4082        df = concat([df1, df2], axis=1)                              4082        df = concat([df1, df2], axis=1)
4083                                                                     4083
4084        with ensure_clean_store(self.path) as store:                 4084        with ensure_clean_store(self.path) as store:
4085                                                                     4085
4086            # exceptions                                             4086            # exceptions
4087            pytest.raises(ValueError, store.append_to_multiple,      4087            pytest.raises(ValueError, store.append_to_multiple,
4088                          {'df1': ['A', 'B'], 'df2': None}, df,      4088                          {'df1': ['A', 'B'], 'df2': None}, df,
4089                          selector='df3')                            4089                          selector='df3')
4090            pytest.raises(ValueError, store.append_to_multiple,      4090            pytest.raises(ValueError, store.append_to_multiple,
4091                          {'df1': None, 'df2': None}, df, selector='4091                          {'df1': None, 'df2': None}, df, selector='
                                                                   df3')                                                                   df3')
4092            pytest.raises(                                           4092            pytest.raises(
4093                ValueError, store.append_to_multiple, 'df1', df, 'df4093                ValueError, store.append_to_multiple, 'df1', df, 'df
                                                                     1')                                                                     1')
4094                                                                     4094
4095            # regular operation                                      4095            # regular operation
4096            store.append_to_multiple(                                4096            store.append_to_multiple(
4097                {'df1': ['A', 'B'], 'df2': None}, df, selector='df1')4097                {'df1': ['A', 'B'], 'df2': None}, df, selector='df1')
4098            result = store.select_as_multiple(                       4098            result = store.select_as_multiple(
4099                ['df1', 'df2'], where=['A>0', 'B>0'], selector='df1')4099                ['df1', 'df2'], where=['A>0', 'B>0'], selector='df1')
4100            expected = df[(df.A > 0) & (df.B > 0)]                   4100            expected = df[(df.A > 0) & (df.B > 0)]
4101            tm.assert_frame_equal(result, expected)                  4101            tm.assert_frame_equal(result, expected)
4102                                                                     4102
4103    def test_append_to_multiple_dropna(self):                        4103    def test_append_to_multiple_dropna(self):
4104        df1 = tm.makeTimeDataFrame()                                 4104        df1 = tm.makeTimeDataFrame()
4105        df2 = tm.makeTimeDataFrame().rename(columns=lambda x: "%s_2"4105        df2 = tm.makeTimeDataFrame().rename(columns=lambda x: "%s_2"
                                                                    % x)                                                                    % x)
4106        df1.iloc[1, df1.columns.get_indexer(['A', 'B'])] = np.nan    4106        df1.iloc[1, df1.columns.get_indexer(['A', 'B'])] = np.nan
4107        df = concat([df1, df2], axis=1)                              4107        df = concat([df1, df2], axis=1)
4108                                                                     4108
4109        with ensure_clean_store(self.path) as store:                 4109        with ensure_clean_store(self.path) as store:
4110                                                                     4110
4111            # dropna=True should guarantee rows are synchronized     4111            # dropna=True should guarantee rows are synchronized
4112            store.append_to_multiple(                                4112            store.append_to_multiple(
4113                {'df1': ['A', 'B'], 'df2': None}, df, selector='df1',4113                {'df1': ['A', 'B'], 'df2': None}, df, selector='df1',
4114                dropna=True)                                         4114                dropna=True)
4115            result = store.select_as_multiple(['df1', 'df2'])        4115            result = store.select_as_multiple(['df1', 'df2'])
4116            expected = df.dropna()                                   4116            expected = df.dropna()
4117            tm.assert_frame_equal(result, expected)                  4117            tm.assert_frame_equal(result, expected)
4118            tm.assert_index_equal(store.select('df1').index,         4118            tm.assert_index_equal(store.select('df1').index,
4119                                  store.select('df2').index)         4119                                  store.select('df2').index)
4120                                                                     4120
4121    @pytest.mark.xfail(run=False,                                    4121    @pytest.mark.xfail(run=False,
4122                       reason="append_to_multiple_dropna_false "     4122                       reason="append_to_multiple_dropna_false "
4123                       "is not raising as failed")                   4123                       "is not raising as failed")
4124    def test_append_to_multiple_dropna_false(self):                  4124    def test_append_to_multiple_dropna_false(self):
4125        df1 = tm.makeTimeDataFrame()                                 4125        df1 = tm.makeTimeDataFrame()
4126        df2 = tm.makeTimeDataFrame().rename(columns=lambda x: "%s_2"4126        df2 = tm.makeTimeDataFrame().rename(columns=lambda x: "%s_2"
                                                                    % x)                                                                    % x)
4127        df1.iloc[1, df1.columns.get_indexer(['A', 'B'])] = np.nan    4127        df1.iloc[1, df1.columns.get_indexer(['A', 'B'])] = np.nan
4128        df = concat([df1, df2], axis=1)                              4128        df = concat([df1, df2], axis=1)
4129                                                                     4129
4130        with ensure_clean_store(self.path) as store:                 4130        with ensure_clean_store(self.path) as store:
4131                                                                     4131
4132            # dropna=False shouldn't synchronize row indexes         4132            # dropna=False shouldn't synchronize row indexes
4133            store.append_to_multiple(                                4133            store.append_to_multiple(
4134                {'df1a': ['A', 'B'], 'df2a': None}, df, selector='df4134                {'df1a': ['A', 'B'], 'df2a': None}, df, selector='df
                                                                    1a',                                                                    1a',
4135                dropna=False)                                        4135                dropna=False)
4136                                                                     4136
4137            with pytest.raises(ValueError):                          4137            with pytest.raises(ValueError):
4138                store.select_as_multiple(['df1a', 'df2a'])           4138                store.select_as_multiple(['df1a', 'df2a'])
4139                                                                     4139
4140            assert not store.select('df1a').index.equals(            4140            assert not store.select('df1a').index.equals(
4141                store.select('df2a').index)                          4141                store.select('df2a').index)
4142                                                                     4142
4143    def test_select_as_multiple(self):                               4143    def test_select_as_multiple(self):
4144                                                                     4144
4145        df1 = tm.makeTimeDataFrame()                                 4145        df1 = tm.makeTimeDataFrame()
4146        df2 = tm.makeTimeDataFrame().rename(columns=lambda x: "%s_2"4146        df2 = tm.makeTimeDataFrame().rename(columns=lambda x: "%s_2"
                                                                    % x)                                                                    % x)
4147        df2['foo'] = 'bar'                                           4147        df2['foo'] = 'bar'
4148                                                                     4148
4149        with ensure_clean_store(self.path) as store:                 4149        with ensure_clean_store(self.path) as store:
4150                                                                     4150
4151            # no tables stored                                       4151            # no tables stored
4152            pytest.raises(Exception, store.select_as_multiple,       4152            pytest.raises(Exception, store.select_as_multiple,
4153                          None, where=['A>0', 'B>0'], selector='df1')4153                          None, where=['A>0', 'B>0'], selector='df1')
4154                                                                     4154
4155            store.append('df1', df1, data_columns=['A', 'B'])        4155            store.append('df1', df1, data_columns=['A', 'B'])
4156            store.append('df2', df2)                                 4156            store.append('df2', df2)
4157                                                                     4157
4158            # exceptions                                             4158            # exceptions
4159            pytest.raises(Exception, store.select_as_multiple,       4159            pytest.raises(Exception, store.select_as_multiple,
4160                          None, where=['A>0', 'B>0'], selector='df1')4160                          None, where=['A>0', 'B>0'], selector='df1')
4161            pytest.raises(Exception, store.select_as_multiple,       4161            pytest.raises(Exception, store.select_as_multiple,
4162                          [None], where=['A>0', 'B>0'], selector='df4162                          [None], where=['A>0', 'B>0'], selector='df
                                                                     1')                                                                     1')
4163            pytest.raises(KeyError, store.select_as_multiple,        4163            pytest.raises(KeyError, store.select_as_multiple,
4164                          ['df1', 'df3'], where=['A>0', 'B>0'],      4164                          ['df1', 'df3'], where=['A>0', 'B>0'],
4165                          selector='df1')                            4165                          selector='df1')
4166            pytest.raises(KeyError, store.select_as_multiple,        4166            pytest.raises(KeyError, store.select_as_multiple,
4167                          ['df3'], where=['A>0', 'B>0'], selector='d4167                          ['df3'], where=['A>0', 'B>0'], selector='d
                                                                    f1')                                                                    f1')
4168            pytest.raises(KeyError, store.select_as_multiple,        4168            pytest.raises(KeyError, store.select_as_multiple,
4169                          ['df1', 'df2'], where=['A>0', 'B>0'],      4169                          ['df1', 'df2'], where=['A>0', 'B>0'],
4170                          selector='df4')                            4170                          selector='df4')
4171                                                                     4171
4172            # default select                                         4172            # default select
4173            result = store.select('df1', ['A>0', 'B>0'])             4173            result = store.select('df1', ['A>0', 'B>0'])
4174            expected = store.select_as_multiple(                     4174            expected = store.select_as_multiple(
4175                ['df1'], where=['A>0', 'B>0'], selector='df1')       4175                ['df1'], where=['A>0', 'B>0'], selector='df1')
4176            tm.assert_frame_equal(result, expected)                  4176            tm.assert_frame_equal(result, expected)
4177            expected = store.select_as_multiple(                     4177            expected = store.select_as_multiple(
4178                'df1', where=['A>0', 'B>0'], selector='df1')         4178                'df1', where=['A>0', 'B>0'], selector='df1')
4179            tm.assert_frame_equal(result, expected)                  4179            tm.assert_frame_equal(result, expected)
4180                                                                     4180
4181            # multiple                                               4181            # multiple
4182            result = store.select_as_multiple(                       4182            result = store.select_as_multiple(
4183                ['df1', 'df2'], where=['A>0', 'B>0'], selector='df1')4183                ['df1', 'df2'], where=['A>0', 'B>0'], selector='df1')
4184            expected = concat([df1, df2], axis=1)                    4184            expected = concat([df1, df2], axis=1)
4185            expected = expected[(expected.A > 0) & (expected.B > 0)] 4185            expected = expected[(expected.A > 0) & (expected.B > 0)]
4186            tm.assert_frame_equal(result, expected)                  4186            tm.assert_frame_equal(result, expected)
4187                                                                     4187
4188            # multiple (diff selector)                               4188            # multiple (diff selector)
4189            result = store.select_as_multiple(                       4189            result = store.select_as_multiple(
4190                ['df1', 'df2'], where='index>df2.index[4]', selector4190                ['df1', 'df2'], where='index>df2.index[4]', selector
                                                                 ='df2')                                                                 ='df2')
4191            expected = concat([df1, df2], axis=1)                    4191            expected = concat([df1, df2], axis=1)
4192            expected = expected[5:]                                  4192            expected = expected[5:]
4193            tm.assert_frame_equal(result, expected)                  4193            tm.assert_frame_equal(result, expected)
4194                                                                     4194
4195            # test excpection for diff rows                          4195            # test excpection for diff rows
4196            store.append('df3', tm.makeTimeDataFrame(nper=50))       4196            store.append('df3', tm.makeTimeDataFrame(nper=50))
4197            pytest.raises(ValueError, store.select_as_multiple,      4197            pytest.raises(ValueError, store.select_as_multiple,
4198                          ['df1', 'df3'], where=['A>0', 'B>0'],      4198                          ['df1', 'df3'], where=['A>0', 'B>0'],
4199                          selector='df1')                            4199                          selector='df1')
4200                                                                     4200
4201    def test_nan_selection_bug_4858(self):                           4201    def test_nan_selection_bug_4858(self):
4202                                                                     4202
4203        # GH 4858; nan selection bug, only works for pytables >= 3.1 4203        # GH 4858; nan selection bug, only works for pytables >= 3.1
4204        if LooseVersion(tables.__version__) < '3.1.0':               4204        if LooseVersion(tables.__version__) < '3.1.0':
4205            pytest.skip('tables version does not support fix for nan4205            pytest.skip('tables version does not support fix for nan
                                                                       '                                                                       '
4206                        'selection bug: GH 4858')                    4206                        'selection bug: GH 4858')
4207                                                                     4207
4208        with ensure_clean_store(self.path) as store:                 4208        with ensure_clean_store(self.path) as store:
4209                                                                     4209
4210            df = DataFrame(dict(cols=range(6), values=range(6)),     4210            df = DataFrame(dict(cols=range(6), values=range(6)),
4211                           dtype='float64')                          4211                           dtype='float64')
4212            df['cols'] = (df['cols'] + 10).apply(str)                4212            df['cols'] = (df['cols'] + 10).apply(str)
4213            df.iloc[0] = np.nan                                      4213            df.iloc[0] = np.nan
4214                                                                     4214
4215            expected = DataFrame(dict(cols=['13.0', '14.0', '15.0'],4215            expected = DataFrame(dict(cols=['13.0', '14.0', '15.0'],
                                                                values=[                                                                values=[
4216                                 3., 4., 5.]), index=[3, 4, 5])      4216                                 3., 4., 5.]), index=[3, 4, 5])
4217                                                                     4217
4218            # write w/o the index on that particular column          4218            # write w/o the index on that particular column
4219            store.append('df', df, data_columns=True, index=['cols'])4219            store.append('df', df, data_columns=True, index=['cols'])
4220            result = store.select('df', where='values>2.0')          4220            result = store.select('df', where='values>2.0')
4221            assert_frame_equal(result, expected)                     4221            assert_frame_equal(result, expected)
4222                                                                     4222
4223    def test_start_stop_table(self):                                 4223    def test_start_stop_table(self):
4224                                                                     4224
4225        with ensure_clean_store(self.path) as store:                 4225        with ensure_clean_store(self.path) as store:
4226                                                                     4226
4227            # table                                                  4227            # table
4228            df = DataFrame(dict(A=np.random.rand(20), B=np.random.ra4228            df = DataFrame(dict(A=np.random.rand(20), B=np.random.ra
                                                                nd(20)))                                                                nd(20)))
4229            store.append('df', df)                                   4229            store.append('df', df)
4230                                                                     4230
4231            result = store.select(                                   4231            result = store.select(
4232                'df', "columns=['A']", start=0, stop=5)              4232                'df', "columns=['A']", start=0, stop=5)
4233            expected = df.loc[0:4, ['A']]                            4233            expected = df.loc[0:4, ['A']]
4234            tm.assert_frame_equal(result, expected)                  4234            tm.assert_frame_equal(result, expected)
4235                                                                     4235
4236            # out of range                                           4236            # out of range
4237            result = store.select(                                   4237            result = store.select(
4238                'df', "columns=['A']", start=30, stop=40)            4238                'df', "columns=['A']", start=30, stop=40)
4239            assert len(result) == 0                                  4239            assert len(result) == 0
4240            expected = df.loc[30:40, ['A']]                          4240            expected = df.loc[30:40, ['A']]
4241            tm.assert_frame_equal(result, expected)                  4241            tm.assert_frame_equal(result, expected)
4242                                                                     4242
4243    def test_start_stop_multiple(self):                              4243    def test_start_stop_multiple(self):
4244                                                                     4244
4245        # GH 16209                                                   4245        # GH 16209
4246        with ensure_clean_store(self.path) as store:                 4246        with ensure_clean_store(self.path) as store:
4247                                                                     4247
4248            df = DataFrame({"foo": [1, 2], "bar": [1, 2]})           4248            df = DataFrame({"foo": [1, 2], "bar": [1, 2]})
4249                                                                     4249
4250            store.append_to_multiple({'selector': ['foo'], 'data': N4250            store.append_to_multiple({'selector': ['foo'], 'data': N
                                                               one}, df,                                                               one}, df,
4251                                     selector='selector')            4251                                     selector='selector')
4252            result = store.select_as_multiple(['selector', 'data'],  4252            result = store.select_as_multiple(['selector', 'data'],
4253                                              selector='selector', s4253                                              selector='selector', s
                                                                 tart=0,                                                                 tart=0,
4254                                              stop=1)                4254                                              stop=1)
4255            expected = df.loc[[0], ['foo', 'bar']]                   4255            expected = df.loc[[0], ['foo', 'bar']]
4256            tm.assert_frame_equal(result, expected)                  4256            tm.assert_frame_equal(result, expected)
4257                                                                     4257
4258    def test_start_stop_fixed(self):                                 4258    def test_start_stop_fixed(self):
4259                                                                     4259
4260        with ensure_clean_store(self.path) as store:                 4260        with ensure_clean_store(self.path) as store:
4261                                                                     4261
4262            # fixed, GH 8287                                         4262            # fixed, GH 8287
4263            df = DataFrame(dict(A=np.random.rand(20),                4263            df = DataFrame(dict(A=np.random.rand(20),
4264                                B=np.random.rand(20)),               4264                                B=np.random.rand(20)),
4265                           index=pd.date_range('20130101', periods=24265                           index=pd.date_range('20130101', periods=2
                                                                     0))                                                                     0))
4266            store.put('df', df)                                      4266            store.put('df', df)
4267                                                                     4267
4268            result = store.select(                                   4268            result = store.select(
4269                'df', start=0, stop=5)                               4269                'df', start=0, stop=5)
4270            expected = df.iloc[0:5, :]                               4270            expected = df.iloc[0:5, :]
4271            tm.assert_frame_equal(result, expected)                  4271            tm.assert_frame_equal(result, expected)
4272                                                                     4272
4273            result = store.select(                                   4273            result = store.select(
4274                'df', start=5, stop=10)                              4274                'df', start=5, stop=10)
4275            expected = df.iloc[5:10, :]                              4275            expected = df.iloc[5:10, :]
4276            tm.assert_frame_equal(result, expected)                  4276            tm.assert_frame_equal(result, expected)
4277                                                                     4277
4278            # out of range                                           4278            # out of range
4279            result = store.select(                                   4279            result = store.select(
4280                'df', start=30, stop=40)                             4280                'df', start=30, stop=40)
4281            expected = df.iloc[30:40, :]                             4281            expected = df.iloc[30:40, :]
4282            tm.assert_frame_equal(result, expected)                  4282            tm.assert_frame_equal(result, expected)
4283                                                                     4283
4284            # series                                                 4284            # series
4285            s = df.A                                                 4285            s = df.A
4286            store.put('s', s)                                        4286            store.put('s', s)
4287            result = store.select('s', start=0, stop=5)              4287            result = store.select('s', start=0, stop=5)
4288            expected = s.iloc[0:5]                                   4288            expected = s.iloc[0:5]
4289            tm.assert_series_equal(result, expected)                 4289            tm.assert_series_equal(result, expected)
4290                                                                     4290
4291            result = store.select('s', start=5, stop=10)             4291            result = store.select('s', start=5, stop=10)
4292            expected = s.iloc[5:10]                                  4292            expected = s.iloc[5:10]
4293            tm.assert_series_equal(result, expected)                 4293            tm.assert_series_equal(result, expected)
4294                                                                     4294
4295            # sparse; not implemented                                4295            # sparse; not implemented
4296            df = tm.makeDataFrame()                                  4296            df = tm.makeDataFrame()
4297            df.iloc[3:5, 1:3] = np.nan                               4297            df.iloc[3:5, 1:3] = np.nan
4298            df.iloc[8:10, -2] = np.nan                               4298            df.iloc[8:10, -2] = np.nan
4299            dfs = df.to_sparse()                                     4299            dfs = df.to_sparse()
4300            store.put('dfs', dfs)                                    4300            store.put('dfs', dfs)
4301            with pytest.raises(NotImplementedError):                 4301            with pytest.raises(NotImplementedError):
4302                store.select('dfs', start=0, stop=5)                 4302                store.select('dfs', start=0, stop=5)
4303                                                                     4303
4304    def test_select_filter_corner(self):                             4304    def test_select_filter_corner(self):
4305                                                                     4305
4306        df = DataFrame(np.random.randn(50, 100))                     4306        df = DataFrame(np.random.randn(50, 100))
4307        df.index = ['%.3d' % c for c in df.index]                    4307        df.index = ['%.3d' % c for c in df.index]
4308        df.columns = ['%.3d' % c for c in df.columns]                4308        df.columns = ['%.3d' % c for c in df.columns]
4309                                                                     4309
4310        with ensure_clean_store(self.path) as store:                 4310        with ensure_clean_store(self.path) as store:
4311            store.put('frame', df, format='table')                   4311            store.put('frame', df, format='table')
4312                                                                     4312
4313            crit = 'columns=df.columns[:75]'                         4313            crit = 'columns=df.columns[:75]'
4314            result = store.select('frame', [crit])                   4314            result = store.select('frame', [crit])
4315            tm.assert_frame_equal(result, df.loc[:, df.columns[:75]])4315            tm.assert_frame_equal(result, df.loc[:, df.columns[:75]])
4316                                                                     4316
4317            crit = 'columns=df.columns[:75:2]'                       4317            crit = 'columns=df.columns[:75:2]'
4318            result = store.select('frame', [crit])                   4318            result = store.select('frame', [crit])
4319            tm.assert_frame_equal(result, df.loc[:, df.columns[:75:24319            tm.assert_frame_equal(result, df.loc[:, df.columns[:75:2
                                                                     ]])                                                                     ]])
4320                                                                     4320
4321    def test_path_pathlib(self):                                     4321    def test_path_pathlib(self):
4322        df = tm.makeDataFrame()                                      4322        df = tm.makeDataFrame()
4323                                                                     4323
4324        result = tm.round_trip_pathlib(                              4324        result = tm.round_trip_pathlib(
4325            lambda p: df.to_hdf(p, 'df'),                            4325            lambda p: df.to_hdf(p, 'df'),
4326            lambda p: pd.read_hdf(p, 'df'))                          4326            lambda p: pd.read_hdf(p, 'df'))
4327        tm.assert_frame_equal(df, result)                            4327        tm.assert_frame_equal(df, result)
4328                                                                     4328
4329    @pytest.mark.xfail(reason='pathlib currently doesnt work with HD4329    @pytest.mark.xfail(reason='pathlib currently doesnt work with HD
                                                                FStore')                                                                FStore')
4330    def test_path_pathlib_hdfstore(self):                            4330    def test_path_pathlib_hdfstore(self):
4331        df = tm.makeDataFrame()                                      4331        df = tm.makeDataFrame()
4332                                                                     4332
4333        def writer(path):                                            4333        def writer(path):
4334            with pd.HDFStore(path) as store:                         4334            with pd.HDFStore(path) as store:
4335                df.to_hdf(store, 'df')                               4335                df.to_hdf(store, 'df')
4336                                                                     4336
4337        def reader(path):                                            4337        def reader(path):
4338            with pd.HDFStore(path) as store:                         4338            with pd.HDFStore(path) as store:
4339                pd.read_hdf(store, 'df')                             4339                pd.read_hdf(store, 'df')
4340        result = tm.round_trip_pathlib(writer, reader)               4340        result = tm.round_trip_pathlib(writer, reader)
4341        tm.assert_frame_equal(df, result)                            4341        tm.assert_frame_equal(df, result)
4342                                                                     4342
4343    def test_pickle_path_localpath(self):                            4343    def test_pickle_path_localpath(self):
4344        df = tm.makeDataFrame()                                      4344        df = tm.makeDataFrame()
4345        result = tm.round_trip_pathlib(                              4345        result = tm.round_trip_pathlib(
4346            lambda p: df.to_hdf(p, 'df'),                            4346            lambda p: df.to_hdf(p, 'df'),
4347            lambda p: pd.read_hdf(p, 'df'))                          4347            lambda p: pd.read_hdf(p, 'df'))
4348        tm.assert_frame_equal(df, result)                            4348        tm.assert_frame_equal(df, result)
4349                                                                     4349
4350    @pytest.mark.xfail(reason='localpath currently doesnt work with 4350    @pytest.mark.xfail(reason='localpath currently doesnt work with 
                                                              HDFStore')                                                              HDFStore')
4351    def test_path_localpath_hdfstore(self):                          4351    def test_path_localpath_hdfstore(self):
4352        df = tm.makeDataFrame()                                      4352        df = tm.makeDataFrame()
4353                                                                     4353
4354        def writer(path):                                            4354        def writer(path):
4355            with pd.HDFStore(path) as store:                         4355            with pd.HDFStore(path) as store:
4356                df.to_hdf(store, 'df')                               4356                df.to_hdf(store, 'df')
4357                                                                     4357
4358        def reader(path):                                            4358        def reader(path):
4359            with pd.HDFStore(path) as store:                         4359            with pd.HDFStore(path) as store:
4360                pd.read_hdf(store, 'df')                             4360                pd.read_hdf(store, 'df')
4361        result = tm.round_trip_localpath(writer, reader)             4361        result = tm.round_trip_localpath(writer, reader)
4362        tm.assert_frame_equal(df, result)                            4362        tm.assert_frame_equal(df, result)
4363                                                                     4363
4364    def _check_roundtrip(self, obj, comparator, compression=False, *4364    def _check_roundtrip(self, obj, comparator, compression=False, *
                                                               *kwargs):                                                               *kwargs):
4365                                                                     4365
4366        options = {}                                                 4366        options = {}
4367        if compression:                                              4367        if compression:
4368            options['complib'] = _default_compressor                 4368            options['complib'] = _default_compressor
4369                                                                     4369
4370        with ensure_clean_store(self.path, 'w', **options) as store: 4370        with ensure_clean_store(self.path, 'w', **options) as store:
4371            store['obj'] = obj                                       4371            store['obj'] = obj
4372            retrieved = store['obj']                                 4372            retrieved = store['obj']
4373            comparator(retrieved, obj, **kwargs)                     4373            comparator(retrieved, obj, **kwargs)
4374                                                                     4374
4375    def _check_double_roundtrip(self, obj, comparator, compression=F4375    def _check_double_roundtrip(self, obj, comparator, compression=F
                                                                   alse,                                                                   alse,
4376                                **kwargs):                           4376                                **kwargs):
4377        options = {}                                                 4377        options = {}
4378        if compression:                                              4378        if compression:
4379            options['complib'] = compression or _default_compressor  4379            options['complib'] = compression or _default_compressor
4380                                                                     4380
4381        with ensure_clean_store(self.path, 'w', **options) as store: 4381        with ensure_clean_store(self.path, 'w', **options) as store:
4382            store['obj'] = obj                                       4382            store['obj'] = obj
4383            retrieved = store['obj']                                 4383            retrieved = store['obj']
4384            comparator(retrieved, obj, **kwargs)                     4384            comparator(retrieved, obj, **kwargs)
4385            store['obj'] = retrieved                                 4385            store['obj'] = retrieved
4386            again = store['obj']                                     4386            again = store['obj']
4387            comparator(again, obj, **kwargs)                         4387            comparator(again, obj, **kwargs)
4388                                                                     4388
4389    def _check_roundtrip_table(self, obj, comparator, compression=Fa4389    def _check_roundtrip_table(self, obj, comparator, compression=Fa
                                                                   lse):                                                                   lse):
4390        options = {}                                                 4390        options = {}
4391        if compression:                                              4391        if compression:
4392            options['complib'] = _default_compressor                 4392            options['complib'] = _default_compressor
4393                                                                     4393
4394        with ensure_clean_store(self.path, 'w', **options) as store: 4394        with ensure_clean_store(self.path, 'w', **options) as store:
4395            store.put('obj', obj, format='table')                    4395            store.put('obj', obj, format='table')
4396            retrieved = store['obj']                                 4396            retrieved = store['obj']
4397                                                                     4397
4398            comparator(retrieved, obj)                               4398            comparator(retrieved, obj)
4399                                                                     4399
4400    def test_multiple_open_close(self):                              4400    def test_multiple_open_close(self):
4401        # gh-4409: open & close multiple times                       4401        # gh-4409: open & close multiple times
4402                                                                     4402
4403        with ensure_clean_path(self.path) as path:                   4403        with ensure_clean_path(self.path) as path:
4404                                                                     4404
4405            df = tm.makeDataFrame()                                  4405            df = tm.makeDataFrame()
4406            df.to_hdf(path, 'df', mode='w', format='table')          4406            df.to_hdf(path, 'df', mode='w', format='table')
4407                                                                     4407
4408            # single                                                 4408            # single
4409            store = HDFStore(path)                                   4409            store = HDFStore(path)
4410            assert 'CLOSED' not in str(store)                        4410            assert 'CLOSED' not in str(store)
4411            assert store.is_open                                     4411            assert store.is_open
4412                                                                     4412
4413            store.close()                                            4413            store.close()
4414            assert 'CLOSED' in str(store)                            4414            assert 'CLOSED' in str(store)
4415            assert not store.is_open                                 4415            assert not store.is_open
4416                                                                     4416
4417        with ensure_clean_path(self.path) as path:                   4417        with ensure_clean_path(self.path) as path:
4418                                                                     4418
4419            if pytables._table_file_open_policy_is_strict:           4419            if pytables._table_file_open_policy_is_strict:
4420                                                                     4420
4421                # multiples                                          4421                # multiples
4422                store1 = HDFStore(path)                              4422                store1 = HDFStore(path)
4423                                                                     4423
4424                def f():                                             4424                def f():
4425                    HDFStore(path)                                   4425                    HDFStore(path)
4426                pytest.raises(ValueError, f)                         4426                pytest.raises(ValueError, f)
4427                store1.close()                                       4427                store1.close()
4428                                                                     4428
4429            else:                                                    4429            else:
4430                                                                     4430
4431                # multiples                                          4431                # multiples
4432                store1 = HDFStore(path)                              4432                store1 = HDFStore(path)
4433                store2 = HDFStore(path)                              4433                store2 = HDFStore(path)
4434                                                                     4434
4435                assert 'CLOSED' not in str(store1)                   4435                assert 'CLOSED' not in str(store1)
4436                assert 'CLOSED' not in str(store2)                   4436                assert 'CLOSED' not in str(store2)
4437                assert store1.is_open                                4437                assert store1.is_open
4438                assert store2.is_open                                4438                assert store2.is_open
4439                                                                     4439
4440                store1.close()                                       4440                store1.close()
4441                assert 'CLOSED' in str(store1)                       4441                assert 'CLOSED' in str(store1)
4442                assert not store1.is_open                            4442                assert not store1.is_open
4443                assert 'CLOSED' not in str(store2)                   4443                assert 'CLOSED' not in str(store2)
4444                assert store2.is_open                                4444                assert store2.is_open
4445                                                                     4445
4446                store2.close()                                       4446                store2.close()
4447                assert 'CLOSED' in str(store1)                       4447                assert 'CLOSED' in str(store1)
4448                assert 'CLOSED' in str(store2)                       4448                assert 'CLOSED' in str(store2)
4449                assert not store1.is_open                            4449                assert not store1.is_open
4450                assert not store2.is_open                            4450                assert not store2.is_open
4451                                                                     4451
4452                # nested close                                       4452                # nested close
4453                store = HDFStore(path, mode='w')                     4453                store = HDFStore(path, mode='w')
4454                store.append('df', df)                               4454                store.append('df', df)
4455                                                                     4455
4456                store2 = HDFStore(path)                              4456                store2 = HDFStore(path)
4457                store2.append('df2', df)                             4457                store2.append('df2', df)
4458                store2.close()                                       4458                store2.close()
4459                assert 'CLOSED' in str(store2)                       4459                assert 'CLOSED' in str(store2)
4460                assert not store2.is_open                            4460                assert not store2.is_open
4461                                                                     4461
4462                store.close()                                        4462                store.close()
4463                assert 'CLOSED' in str(store)                        4463                assert 'CLOSED' in str(store)
4464                assert not store.is_open                             4464                assert not store.is_open
4465                                                                     4465
4466                # double closing                                     4466                # double closing
4467                store = HDFStore(path, mode='w')                     4467                store = HDFStore(path, mode='w')
4468                store.append('df', df)                               4468                store.append('df', df)
4469                                                                     4469
4470                store2 = HDFStore(path)                              4470                store2 = HDFStore(path)
4471                store.close()                                        4471                store.close()
4472                assert 'CLOSED' in str(store)                        4472                assert 'CLOSED' in str(store)
4473                assert not store.is_open                             4473                assert not store.is_open
4474                                                                     4474
4475                store2.close()                                       4475                store2.close()
4476                assert 'CLOSED' in str(store2)                       4476                assert 'CLOSED' in str(store2)
4477                assert not store2.is_open                            4477                assert not store2.is_open
4478                                                                     4478
4479        # ops on a closed store                                      4479        # ops on a closed store
4480        with ensure_clean_path(self.path) as path:                   4480        with ensure_clean_path(self.path) as path:
4481                                                                     4481
4482            df = tm.makeDataFrame()                                  4482            df = tm.makeDataFrame()
4483            df.to_hdf(path, 'df', mode='w', format='table')          4483            df.to_hdf(path, 'df', mode='w', format='table')
4484                                                                     4484
4485            store = HDFStore(path)                                   4485            store = HDFStore(path)
4486            store.close()                                            4486            store.close()
4487                                                                     4487
4488            pytest.raises(ClosedFileError, store.keys)               4488            pytest.raises(ClosedFileError, store.keys)
4489            pytest.raises(ClosedFileError, lambda: 'df' in store)    4489            pytest.raises(ClosedFileError, lambda: 'df' in store)
4490            pytest.raises(ClosedFileError, lambda: len(store))       4490            pytest.raises(ClosedFileError, lambda: len(store))
4491            pytest.raises(ClosedFileError, lambda: store['df'])      4491            pytest.raises(ClosedFileError, lambda: store['df'])
4492            pytest.raises(ClosedFileError, lambda: store.df)         4492            pytest.raises(ClosedFileError, lambda: store.df)
4493            pytest.raises(ClosedFileError, store.select, 'df')       4493            pytest.raises(ClosedFileError, store.select, 'df')
4494            pytest.raises(ClosedFileError, store.get, 'df')          4494            pytest.raises(ClosedFileError, store.get, 'df')
4495            pytest.raises(ClosedFileError, store.append, 'df2', df)  4495            pytest.raises(ClosedFileError, store.append, 'df2', df)
4496            pytest.raises(ClosedFileError, store.put, 'df3', df)     4496            pytest.raises(ClosedFileError, store.put, 'df3', df)
4497            pytest.raises(ClosedFileError, store.get_storer, 'df2')  4497            pytest.raises(ClosedFileError, store.get_storer, 'df2')
4498            pytest.raises(ClosedFileError, store.remove, 'df2')      4498            pytest.raises(ClosedFileError, store.remove, 'df2')
4499                                                                     4499
4500            def f():                                                 4500            def f():
4501                store.select('df')                                   4501                store.select('df')
4502            tm.assert_raises_regex(ClosedFileError, 'file is not ope4502            tm.assert_raises_regex(ClosedFileError, 'file is not ope
                                                                  n', f)                                                                  n', f)
4503                                                                     4503
4504    def test_pytables_native_read(self):                             4504    def test_pytables_native_read(self):
4505                                                                     4505
4506        with ensure_clean_store(                                     4506        with ensure_clean_store(
4507                tm.get_data_path('legacy_hdf/pytables_native.h5'),   4507                tm.get_data_path('legacy_hdf/pytables_native.h5'),
4508                mode='r') as store:                                  4508                mode='r') as store:
4509            d2 = store['detector/readout']                           4509            d2 = store['detector/readout']
4510            assert isinstance(d2, DataFrame)                         4510            assert isinstance(d2, DataFrame)
4511                                                                     4511
4512    def test_pytables_native2_read(self):                            4512    def test_pytables_native2_read(self):
4513        # fails on win/3.5 oddly                                     4513        # fails on win/3.5 oddly
4514        if PY35 and is_platform_windows():                           4514        if PY35 and is_platform_windows():
4515            pytest.skip("native2 read fails oddly on windows / 3.5") 4515            pytest.skip("native2 read fails oddly on windows / 3.5")
4516                                                                     4516
4517        with ensure_clean_store(                                     4517        with ensure_clean_store(
4518                tm.get_data_path('legacy_hdf/pytables_native2.h5'),  4518                tm.get_data_path('legacy_hdf/pytables_native2.h5'),
4519                mode='r') as store:                                  4519                mode='r') as store:
4520            str(store)                                               4520            str(store)
4521            d1 = store['detector']                                   4521            d1 = store['detector']
4522            assert isinstance(d1, DataFrame)                         4522            assert isinstance(d1, DataFrame)
4523                                                                     4523
4524    def test_legacy_table_read(self):                                4524    def test_legacy_table_read(self):
4525        # legacy table types                                         4525        # legacy table types
4526        with ensure_clean_store(                                     4526        with ensure_clean_store(
4527                tm.get_data_path('legacy_hdf/legacy_table.h5'),      4527                tm.get_data_path('legacy_hdf/legacy_table.h5'),
4528                mode='r') as store:                                  4528                mode='r') as store:
4529                                                                     4529
4530            with catch_warnings(record=True):                        4530            with catch_warnings(record=True):
4531                store.select('df1')                                  4531                store.select('df1')
4532                store.select('df2')                                  4532                store.select('df2')
4533                store.select('wp1')                                  4533                store.select('wp1')
4534                                                                     4534
4535                # force the frame                                    4535                # force the frame
4536                store.select('df2', typ='legacy_frame')              4536                store.select('df2', typ='legacy_frame')
4537                                                                     4537
4538                # old version warning                                4538                # old version warning
4539                pytest.raises(                                       4539                pytest.raises(
4540                    Exception, store.select, 'wp1', 'minor_axis=B')  4540                    Exception, store.select, 'wp1', 'minor_axis=B')
4541                                                                     4541
4542                df2 = store.select('df2')                            4542                df2 = store.select('df2')
4543                result = store.select('df2', 'index>df2.index[2]')   4543                result = store.select('df2', 'index>df2.index[2]')
4544                expected = df2[df2.index > df2.index[2]]             4544                expected = df2[df2.index > df2.index[2]]
4545                assert_frame_equal(expected, result)                 4545                assert_frame_equal(expected, result)
4546                                                                     4546
4547    def test_legacy_0_10_read(self):                                 4547    def test_legacy_0_10_read(self):
4548        # legacy from 0.10                                           4548        # legacy from 0.10
4549        with catch_warnings(record=True):                            4549        with catch_warnings(record=True):
4550            path = tm.get_data_path('legacy_hdf/legacy_0.10.h5')     4550            path = tm.get_data_path('legacy_hdf/legacy_0.10.h5')
4551            with ensure_clean_store(path, mode='r') as store:        4551            with ensure_clean_store(path, mode='r') as store:
4552                str(store)                                           4552                str(store)
4553                for k in store.keys():                               4553                for k in store.keys():
4554                    store.select(k)                                  4554                    store.select(k)
4555                                                                     4555
4556    def test_legacy_0_11_read(self):                                 4556    def test_legacy_0_11_read(self):
4557        # legacy from 0.11                                           4557        # legacy from 0.11
4558        path = os.path.join('legacy_hdf', 'legacy_table_0.11.h5')    4558        path = os.path.join('legacy_hdf', 'legacy_table_0.11.h5')
4559        with ensure_clean_store(tm.get_data_path(path), mode='r') as4559        with ensure_clean_store(tm.get_data_path(path), mode='r') as
                                                                  store:                                                                  store:
4560            str(store)                                               4560            str(store)
4561            assert 'df' in store                                     4561            assert 'df' in store
4562            assert 'df1' in store                                    4562            assert 'df1' in store
4563            assert 'mi' in store                                     4563            assert 'mi' in store
4564            df = store.select('df')                                  4564            df = store.select('df')
4565            df1 = store.select('df1')                                4565            df1 = store.select('df1')
4566            mi = store.select('mi')                                  4566            mi = store.select('mi')
4567            assert isinstance(df, DataFrame)                         4567            assert isinstance(df, DataFrame)
4568            assert isinstance(df1, DataFrame)                        4568            assert isinstance(df1, DataFrame)
4569            assert isinstance(mi, DataFrame)                         4569            assert isinstance(mi, DataFrame)
4570                                                                     4570
4571    def test_copy(self):                                             4571    def test_copy(self):
4572                                                                     4572
4573        with catch_warnings(record=True):                            4573        with catch_warnings(record=True):
4574                                                                     4574
4575            def do_copy(f=None, new_f=None, keys=None,               4575            def do_copy(f=None, new_f=None, keys=None,
4576                        propindexes=True, **kwargs):                 4576                        propindexes=True, **kwargs):
4577                try:                                                 4577                try:
4578                    if f is None:                                    4578                    if f is None:
4579                        f = tm.get_data_path(os.path.join('legacy_hd4579                        f = tm.get_data_path(os.path.join('legacy_hd
                                                                     f',                                                                     f',
4580                                                          'legacy_0.4580                                                          'legacy_0.
                                                                10.h5'))                                                                10.h5'))
4581                                                                     4581
4582                    store = HDFStore(f, 'r')                         4582                    store = HDFStore(f, 'r')
4583                                                                     4583
4584                    if new_f is None:                                4584                    if new_f is None:
4585                        import tempfile                              4585                        import tempfile
4586                        fd, new_f = tempfile.mkstemp()               4586                        fd, new_f = tempfile.mkstemp()
4587                                                                     4587
4588                    tstore = store.copy(                             4588                    tstore = store.copy(
4589                        new_f, keys=keys, propindexes=propindexes, *4589                        new_f, keys=keys, propindexes=propindexes, *
                                                                *kwargs)                                                                *kwargs)
4590                                                                     4590
4591                    # check keys                                     4591                    # check keys
4592                    if keys is None:                                 4592                    if keys is None:
4593                        keys = store.keys()                          4593                        keys = store.keys()
4594                    assert set(keys) == set(tstore.keys())           4594                    assert set(keys) == set(tstore.keys())
4595                                                                     4595
4596                    # check indicies & nrows                         4596                    # check indicies & nrows
4597                    for k in tstore.keys():                          4597                    for k in tstore.keys():
4598                        if tstore.get_storer(k).is_table:            4598                        if tstore.get_storer(k).is_table:
4599                            new_t = tstore.get_storer(k)             4599                            new_t = tstore.get_storer(k)
4600                            orig_t = store.get_storer(k)             4600                            orig_t = store.get_storer(k)
4601                                                                     4601
4602                            assert orig_t.nrows == new_t.nrows       4602                            assert orig_t.nrows == new_t.nrows
4603                                                                     4603
4604                            # check propindixes                      4604                            # check propindixes
4605                            if propindexes:                          4605                            if propindexes:
4606                                for a in orig_t.axes:                4606                                for a in orig_t.axes:
4607                                    if a.is_indexed:                 4607                                    if a.is_indexed:
4608                                        assert new_t[a.name].is_inde4608                                        assert new_t[a.name].is_inde
                                                                     xed                                                                     xed
4609                                                                     4609
4610                finally:                                             4610                finally:
4611                    safe_close(store)                                4611                    safe_close(store)
4612                    safe_close(tstore)                               4612                    safe_close(tstore)
4613                    try:                                             4613                    try:
4614                        os.close(fd)                                 4614                        os.close(fd)
4615                    except:                                          4615                    except:
4616                        pass                                         4616                        pass
4617                    safe_remove(new_f)                               4617                    safe_remove(new_f)
4618                                                                     4618
4619            do_copy()                                                4619            do_copy()
4620            do_copy(keys=['/a', '/b', '/df1_mixed'])                 4620            do_copy(keys=['/a', '/b', '/df1_mixed'])
4621            do_copy(propindexes=False)                               4621            do_copy(propindexes=False)
4622                                                                     4622
4623            # new table                                              4623            # new table
4624            df = tm.makeDataFrame()                                  4624            df = tm.makeDataFrame()
4625                                                                     4625
4626            try:                                                     4626            try:
4627                path = create_tempfile(self.path)                    4627                path = create_tempfile(self.path)
4628                st = HDFStore(path)                                  4628                st = HDFStore(path)
4629                st.append('df', df, data_columns=['A'])              4629                st.append('df', df, data_columns=['A'])
4630                st.close()                                           4630                st.close()
4631                do_copy(f=path)                                      4631                do_copy(f=path)
4632                do_copy(f=path, propindexes=False)                   4632                do_copy(f=path, propindexes=False)
4633            finally:                                                 4633            finally:
4634                safe_remove(path)                                    4634                safe_remove(path)
4635                                                                     4635
4636    def test_legacy_table_write(self):                               4636    def test_legacy_table_write(self):
4637        pytest.skip("cannot write legacy tables")                    4637        pytest.skip("cannot write legacy tables")
4638                                                                     4638
4639        store = HDFStore(tm.get_data_path(                           4639        store = HDFStore(tm.get_data_path(
4640            'legacy_hdf/legacy_table_%s.h5' % pandas.__version__), '4640            'legacy_hdf/legacy_table_%s.h5' % pandas.__version__), '
                                                                     a')                                                                     a')
4641                                                                     4641
4642        df = tm.makeDataFrame()                                      4642        df = tm.makeDataFrame()
4643        with catch_warnings(record=True):                            4643        with catch_warnings(record=True):
4644            wp = tm.makePanel()                                      4644            wp = tm.makePanel()
4645                                                                     4645
4646        index = MultiIndex(levels=[['foo', 'bar', 'baz', 'qux'],     4646        index = MultiIndex(levels=[['foo', 'bar', 'baz', 'qux'],
4647                                   ['one', 'two', 'three']],         4647                                   ['one', 'two', 'three']],
4648                           labels=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3],   4648                           labels=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3],
4649                                   [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],  4649                                   [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],
4650                           names=['foo', 'bar'])                     4650                           names=['foo', 'bar'])
4651        df = DataFrame(np.random.randn(10, 3), index=index,          4651        df = DataFrame(np.random.randn(10, 3), index=index,
4652                       columns=['A', 'B', 'C'])                      4652                       columns=['A', 'B', 'C'])
4653        store.append('mi', df)                                       4653        store.append('mi', df)
4654                                                                     4654
4655        df = DataFrame(dict(A='foo', B='bar'), index=lrange(10))     4655        df = DataFrame(dict(A='foo', B='bar'), index=lrange(10))
4656        store.append('df', df, data_columns=['B'], min_itemsize={'A'4656        store.append('df', df, data_columns=['B'], min_itemsize={'A'
                                                                 : 200})                                                                 : 200})
4657        store.append('wp', wp)                                       4657        store.append('wp', wp)
4658                                                                     4658
4659        store.close()                                                4659        store.close()
4660                                                                     4660
4661    def test_store_datetime_fractional_secs(self):                   4661    def test_store_datetime_fractional_secs(self):
4662                                                                     4662
4663        with ensure_clean_store(self.path) as store:                 4663        with ensure_clean_store(self.path) as store:
4664            dt = datetime.datetime(2012, 1, 2, 3, 4, 5, 123456)      4664            dt = datetime.datetime(2012, 1, 2, 3, 4, 5, 123456)
4665            series = Series([0], [dt])                               4665            series = Series([0], [dt])
4666            store['a'] = series                                      4666            store['a'] = series
4667            assert store['a'].index[0] == dt                         4667            assert store['a'].index[0] == dt
4668                                                                     4668
4669    def test_tseries_indices_series(self):                           4669    def test_tseries_indices_series(self):
4670                                                                     4670
4671        with ensure_clean_store(self.path) as store:                 4671        with ensure_clean_store(self.path) as store:
4672            idx = tm.makeDateIndex(10)                               4672            idx = tm.makeDateIndex(10)
4673            ser = Series(np.random.randn(len(idx)), idx)             4673            ser = Series(np.random.randn(len(idx)), idx)
4674            store['a'] = ser                                         4674            store['a'] = ser
4675            result = store['a']                                      4675            result = store['a']
4676                                                                     4676
4677            tm.assert_series_equal(result, ser)                      4677            tm.assert_series_equal(result, ser)
4678            assert result.index.freq == ser.index.freq               4678            assert result.index.freq == ser.index.freq
4679            tm.assert_class_equal(result.index, ser.index, obj="seri4679            tm.assert_class_equal(result.index, ser.index, obj="seri
                                                              es index")                                                              es index")
4680                                                                     4680
4681            idx = tm.makePeriodIndex(10)                             4681            idx = tm.makePeriodIndex(10)
4682            ser = Series(np.random.randn(len(idx)), idx)             4682            ser = Series(np.random.randn(len(idx)), idx)
4683            store['a'] = ser                                         4683            store['a'] = ser
4684            result = store['a']                                      4684            result = store['a']
4685                                                                     4685
4686            tm.assert_series_equal(result, ser)                      4686            tm.assert_series_equal(result, ser)
4687            assert result.index.freq == ser.index.freq               4687            assert result.index.freq == ser.index.freq
4688            tm.assert_class_equal(result.index, ser.index, obj="seri4688            tm.assert_class_equal(result.index, ser.index, obj="seri
                                                              es index")                                                              es index")
4689                                                                     4689
4690    def test_tseries_indices_frame(self):                            4690    def test_tseries_indices_frame(self):
4691                                                                     4691
4692        with ensure_clean_store(self.path) as store:                 4692        with ensure_clean_store(self.path) as store:
4693            idx = tm.makeDateIndex(10)                               4693            idx = tm.makeDateIndex(10)
4694            df = DataFrame(np.random.randn(len(idx), 3), index=idx)  4694            df = DataFrame(np.random.randn(len(idx), 3), index=idx)
4695            store['a'] = df                                          4695            store['a'] = df
4696            result = store['a']                                      4696            result = store['a']
4697                                                                     4697
4698            assert_frame_equal(result, df)                           4698            assert_frame_equal(result, df)
4699            assert result.index.freq == df.index.freq                4699            assert result.index.freq == df.index.freq
4700            tm.assert_class_equal(result.index, df.index,            4700            tm.assert_class_equal(result.index, df.index,
4701                                  obj="dataframe index")             4701                                  obj="dataframe index")
4702                                                                     4702
4703            idx = tm.makePeriodIndex(10)                             4703            idx = tm.makePeriodIndex(10)
4704            df = DataFrame(np.random.randn(len(idx), 3), idx)        4704            df = DataFrame(np.random.randn(len(idx), 3), idx)
4705            store['a'] = df                                          4705            store['a'] = df
4706            result = store['a']                                      4706            result = store['a']
4707                                                                     4707
4708            assert_frame_equal(result, df)                           4708            assert_frame_equal(result, df)
4709            assert result.index.freq == df.index.freq                4709            assert result.index.freq == df.index.freq
4710            tm.assert_class_equal(result.index, df.index,            4710            tm.assert_class_equal(result.index, df.index,
4711                                  obj="dataframe index")             4711                                  obj="dataframe index")
4712                                                                     4712
4713    def test_unicode_index(self):                                    4713    def test_unicode_index(self):
4714                                                                     4714
4715        unicode_values = [u('\u03c3'), u('\u03c3\u03c3')]            4715        unicode_values = [u('\u03c3'), u('\u03c3\u03c3')]
4716                                                                     4716
4717        # PerformanceWarning                                         4717        # PerformanceWarning
4718        with catch_warnings(record=True):                            4718        with catch_warnings(record=True):
4719            s = Series(np.random.randn(len(unicode_values)), unicode4719            s = Series(np.random.randn(len(unicode_values)), unicode
                                                                _values)                                                                _values)
4720            self._check_roundtrip(s, tm.assert_series_equal)         4720            self._check_roundtrip(s, tm.assert_series_equal)
4721                                                                     4721
4722    def test_unicode_longer_encoded(self):                           4722    def test_unicode_longer_encoded(self):
4723        # GH 11234                                                   4723        # GH 11234
4724        char = '\u0394'                                              4724        char = '\u0394'
4725        df = pd.DataFrame({'A': [char]})                             4725        df = pd.DataFrame({'A': [char]})
4726        with ensure_clean_store(self.path) as store:                 4726        with ensure_clean_store(self.path) as store:
4727            store.put('df', df, format='table', encoding='utf-8')    4727            store.put('df', df, format='table', encoding='utf-8')
4728            result = store.get('df')                                 4728            result = store.get('df')
4729            tm.assert_frame_equal(result, df)                        4729            tm.assert_frame_equal(result, df)
4730                                                                     4730
4731        df = pd.DataFrame({'A': ['a', char], 'B': ['b', 'b']})       4731        df = pd.DataFrame({'A': ['a', char], 'B': ['b', 'b']})
4732        with ensure_clean_store(self.path) as store:                 4732        with ensure_clean_store(self.path) as store:
4733            store.put('df', df, format='table', encoding='utf-8')    4733            store.put('df', df, format='table', encoding='utf-8')
4734            result = store.get('df')                                 4734            result = store.get('df')
4735            tm.assert_frame_equal(result, df)                        4735            tm.assert_frame_equal(result, df)
4736                                                                     4736
4737    def test_store_datetime_mixed(self):                             4737    def test_store_datetime_mixed(self):
4738                                                                     4738
4739        df = DataFrame(                                              4739        df = DataFrame(
4740            {'a': [1, 2, 3], 'b': [1., 2., 3.], 'c': ['a', 'b', 'c']4740            {'a': [1, 2, 3], 'b': [1., 2., 3.], 'c': ['a', 'b', 'c']
                                                                      })                                                                      })
4741        ts = tm.makeTimeSeries()                                     4741        ts = tm.makeTimeSeries()
4742        df['d'] = ts.index[:3]                                       4742        df['d'] = ts.index[:3]
4743        self._check_roundtrip(df, tm.assert_frame_equal)             4743        self._check_roundtrip(df, tm.assert_frame_equal)
4744                                                                     4744
4745    # def test_cant_write_multiindex_table(self):                    4745    # def test_cant_write_multiindex_table(self):
4746    #    # for now, #1848                                            4746    #    # for now, #1848
4747    #    df = DataFrame(np.random.randn(10, 4),                      4747    #    df = DataFrame(np.random.randn(10, 4),
4748    #                   index=[np.arange(5).repeat(2),               4748    #                   index=[np.arange(5).repeat(2),
4749    #                          np.tile(np.arange(2), 5)])            4749    #                          np.tile(np.arange(2), 5)])
4750                                                                     4750
4751    #    pytest.raises(Exception, store.put, 'foo', df, format='tabl4751    #    pytest.raises(Exception, store.put, 'foo', df, format='tabl
                                                                     e')                                                                     e')
4752                                                                     4752
4753    def test_append_with_diff_col_name_types_raises_value_error(self4753    def test_append_with_diff_col_name_types_raises_value_error(self
                                                                      ):                                                                      ):
4754        df = DataFrame(np.random.randn(10, 1))                       4754        df = DataFrame(np.random.randn(10, 1))
4755        df2 = DataFrame({'a': np.random.randn(10)})                  4755        df2 = DataFrame({'a': np.random.randn(10)})
4756        df3 = DataFrame({(1, 2): np.random.randn(10)})               4756        df3 = DataFrame({(1, 2): np.random.randn(10)})
4757        df4 = DataFrame({('1', 2): np.random.randn(10)})             4757        df4 = DataFrame({('1', 2): np.random.randn(10)})
4758        df5 = DataFrame({('1', 2, object): np.random.randn(10)})     4758        df5 = DataFrame({('1', 2, object): np.random.randn(10)})
4759                                                                     4759
4760        with ensure_clean_store(self.path) as store:                 4760        with ensure_clean_store(self.path) as store:
4761            name = 'df_%s' % tm.rands(10)                            4761            name = 'df_%s' % tm.rands(10)
4762            store.append(name, df)                                   4762            store.append(name, df)
4763                                                                     4763
4764            for d in (df2, df3, df4, df5):                           4764            for d in (df2, df3, df4, df5):
4765                with pytest.raises(ValueError):                      4765                with pytest.raises(ValueError):
4766                    store.append(name, d)                            4766                    store.append(name, d)
4767                                                                     4767
4768    def test_query_with_nested_special_character(self):              4768    def test_query_with_nested_special_character(self):
4769        df = DataFrame({'a': ['a', 'a', 'c', 'b',                    4769        df = DataFrame({'a': ['a', 'a', 'c', 'b',
4770                              'test & test', 'c', 'b', 'e'],         4770                              'test & test', 'c', 'b', 'e'],
4771                        'b': [1, 2, 3, 4, 5, 6, 7, 8]})              4771                        'b': [1, 2, 3, 4, 5, 6, 7, 8]})
4772        expected = df[df.a == 'test & test']                         4772        expected = df[df.a == 'test & test']
4773        with ensure_clean_store(self.path) as store:                 4773        with ensure_clean_store(self.path) as store:
4774            store.append('test', df, format='table', data_columns=Tr4774            store.append('test', df, format='table', data_columns=Tr
                                                                     ue)                                                                     ue)
4775            result = store.select('test', 'a = "test & test"')       4775            result = store.select('test', 'a = "test & test"')
4776        tm.assert_frame_equal(expected, result)                      4776        tm.assert_frame_equal(expected, result)
4777                                                                     4777
4778    def test_categorical(self):                                      4778    def test_categorical(self):
4779                                                                     4779
4780        with ensure_clean_store(self.path) as store:                 4780        with ensure_clean_store(self.path) as store:
4781                                                                     4781
4782            # Basic                                                  4782            # Basic
4783            _maybe_remove(store, 's')                                4783            _maybe_remove(store, 's')
4784            s = Series(Categorical(['a', 'b', 'b', 'a', 'a', 'c'], c4784            s = Series(Categorical(['a', 'b', 'b', 'a', 'a', 'c'], c
                                                             ategories=[                                                             ategories=[
4785                       'a', 'b', 'c', 'd'], ordered=False))          4785                       'a', 'b', 'c', 'd'], ordered=False))
4786            store.append('s', s, format='table')                     4786            store.append('s', s, format='table')
4787            result = store.select('s')                               4787            result = store.select('s')
4788            tm.assert_series_equal(s, result)                        4788            tm.assert_series_equal(s, result)
4789                                                                     4789
4790            _maybe_remove(store, 's_ordered')                        4790            _maybe_remove(store, 's_ordered')
4791            s = Series(Categorical(['a', 'b', 'b', 'a', 'a', 'c'], c4791            s = Series(Categorical(['a', 'b', 'b', 'a', 'a', 'c'], c
                                                             ategories=[                                                             ategories=[
4792                       'a', 'b', 'c', 'd'], ordered=True))           4792                       'a', 'b', 'c', 'd'], ordered=True))
4793            store.append('s_ordered', s, format='table')             4793            store.append('s_ordered', s, format='table')
4794            result = store.select('s_ordered')                       4794            result = store.select('s_ordered')
4795            tm.assert_series_equal(s, result)                        4795            tm.assert_series_equal(s, result)
4796                                                                     4796
4797            _maybe_remove(store, 'df')                               4797            _maybe_remove(store, 'df')
4798                                                                     4798
4799            df = DataFrame({"s": s, "vals": [1, 2, 3, 4, 5, 6]})     4799            df = DataFrame({"s": s, "vals": [1, 2, 3, 4, 5, 6]})
4800            store.append('df', df, format='table')                   4800            store.append('df', df, format='table')
4801            result = store.select('df')                              4801            result = store.select('df')
4802            tm.assert_frame_equal(result, df)                        4802            tm.assert_frame_equal(result, df)
4803                                                                     4803
4804            # Dtypes                                                 4804            # Dtypes
4805            s = Series([1, 1, 2, 2, 3, 4, 5]).astype('category')     4805            s = Series([1, 1, 2, 2, 3, 4, 5]).astype('category')
4806            store.append('si', s)                                    4806            store.append('si', s)
4807            result = store.select('si')                              4807            result = store.select('si')
4808            tm.assert_series_equal(result, s)                        4808            tm.assert_series_equal(result, s)
4809                                                                     4809
4810            s = Series([1, 1, np.nan, 2, 3, 4, 5]).astype('category')4810            s = Series([1, 1, np.nan, 2, 3, 4, 5]).astype('category')
4811            store.append('si2', s)                                   4811            store.append('si2', s)
4812            result = store.select('si2')                             4812            result = store.select('si2')
4813            tm.assert_series_equal(result, s)                        4813            tm.assert_series_equal(result, s)
4814                                                                     4814
4815            # Multiple                                               4815            # Multiple
4816            df2 = df.copy()                                          4816            df2 = df.copy()
4817            df2['s2'] = Series(list('abcdefg')).astype('category')   4817            df2['s2'] = Series(list('abcdefg')).astype('category')
4818            store.append('df2', df2)                                 4818            store.append('df2', df2)
4819            result = store.select('df2')                             4819            result = store.select('df2')
4820            tm.assert_frame_equal(result, df2)                       4820            tm.assert_frame_equal(result, df2)
4821                                                                     4821
4822            # Make sure the metadata is OK                           4822            # Make sure the metadata is OK
4823            assert '/df2   ' in str(store)                           4823            assert '/df2   ' in str(store)
4824            assert '/df2/meta/values_block_0/meta' in str(store)     4824            assert '/df2/meta/values_block_0/meta' in str(store)
4825            assert '/df2/meta/values_block_1/meta' in str(store)     4825            assert '/df2/meta/values_block_1/meta' in str(store)
4826                                                                     4826
4827            # unordered                                              4827            # unordered
4828            s = Series(Categorical(['a', 'b', 'b', 'a', 'a', 'c'], c4828            s = Series(Categorical(['a', 'b', 'b', 'a', 'a', 'c'], c
                                                             ategories=[                                                             ategories=[
4829                       'a', 'b', 'c', 'd'], ordered=False))          4829                       'a', 'b', 'c', 'd'], ordered=False))
4830            store.append('s2', s, format='table')                    4830            store.append('s2', s, format='table')
4831            result = store.select('s2')                              4831            result = store.select('s2')
4832            tm.assert_series_equal(result, s)                        4832            tm.assert_series_equal(result, s)
4833                                                                     4833
4834            # Query                                                  4834            # Query
4835            store.append('df3', df, data_columns=['s'])              4835            store.append('df3', df, data_columns=['s'])
4836            expected = df[df.s.isin(['b', 'c'])]                     4836            expected = df[df.s.isin(['b', 'c'])]
4837            result = store.select('df3', where=['s in ["b","c"]'])   4837            result = store.select('df3', where=['s in ["b","c"]'])
4838            tm.assert_frame_equal(result, expected)                  4838            tm.assert_frame_equal(result, expected)
4839                                                                     4839
4840            expected = df[df.s.isin(['b', 'c'])]                     4840            expected = df[df.s.isin(['b', 'c'])]
4841            result = store.select('df3', where=['s = ["b","c"]'])    4841            result = store.select('df3', where=['s = ["b","c"]'])
4842            tm.assert_frame_equal(result, expected)                  4842            tm.assert_frame_equal(result, expected)
4843                                                                     4843
4844            expected = df[df.s.isin(['d'])]                          4844            expected = df[df.s.isin(['d'])]
4845            result = store.select('df3', where=['s in ["d"]'])       4845            result = store.select('df3', where=['s in ["d"]'])
4846            tm.assert_frame_equal(result, expected)                  4846            tm.assert_frame_equal(result, expected)
4847                                                                     4847
4848            expected = df[df.s.isin(['f'])]                          4848            expected = df[df.s.isin(['f'])]
4849            result = store.select('df3', where=['s in ["f"]'])       4849            result = store.select('df3', where=['s in ["f"]'])
4850            tm.assert_frame_equal(result, expected)                  4850            tm.assert_frame_equal(result, expected)
4851                                                                     4851
4852            # Appending with same categories is ok                   4852            # Appending with same categories is ok
4853            store.append('df3', df)                                  4853            store.append('df3', df)
4854                                                                     4854
4855            df = concat([df, df])                                    4855            df = concat([df, df])
4856            expected = df[df.s.isin(['b', 'c'])]                     4856            expected = df[df.s.isin(['b', 'c'])]
4857            result = store.select('df3', where=['s in ["b","c"]'])   4857            result = store.select('df3', where=['s in ["b","c"]'])
4858            tm.assert_frame_equal(result, expected)                  4858            tm.assert_frame_equal(result, expected)
4859                                                                     4859
4860            # Appending must have the same categories                4860            # Appending must have the same categories
4861            df3 = df.copy()                                          4861            df3 = df.copy()
4862            df3['s'].cat.remove_unused_categories(inplace=True)      4862            df3['s'].cat.remove_unused_categories(inplace=True)
4863                                                                     4863
4864            with pytest.raises(ValueError):                          4864            with pytest.raises(ValueError):
4865                store.append('df3', df3)                             4865                store.append('df3', df3)
4866                                                                     4866
4867            # Remove, and make sure meta data is removed (its a recu4867            # Remove, and make sure meta data is removed (its a recu
                                                                   rsive                                                                   rsive
4868            # removal so should be).                                 4868            # removal so should be).
4869            result = store.select('df3/meta/s/meta')                 4869            result = store.select('df3/meta/s/meta')
4870            assert result is not None                                4870            assert result is not None
4871            store.remove('df3')                                      4871            store.remove('df3')
4872                                                                     4872
4873            with pytest.raises(KeyError):                            4873            with pytest.raises(KeyError):
4874                store.select('df3/meta/s/meta')                      4874                store.select('df3/meta/s/meta')
4875                                                                     4875
4876    def test_categorical_conversion(self):                           4876    def test_categorical_conversion(self):
4877                                                                     4877
4878        # GH13322                                                    4878        # GH13322
4879        # Check that read_hdf with categorical columns doesn't retur4879        # Check that read_hdf with categorical columns doesn't retur
                                                               n rows if                                                               n rows if
4880        # where criteria isn't met.                                  4880        # where criteria isn't met.
4881        obsids = ['ESP_012345_6789', 'ESP_987654_3210']              4881        obsids = ['ESP_012345_6789', 'ESP_987654_3210']
4882        imgids = ['APF00006np', 'APF0001imm']                        4882        imgids = ['APF00006np', 'APF0001imm']
4883        data = [4.3, 9.8]                                            4883        data = [4.3, 9.8]
4884                                                                     4884
4885        # Test without categories                                    4885        # Test without categories
4886        df = DataFrame(dict(obsids=obsids, imgids=imgids, data=data))4886        df = DataFrame(dict(obsids=obsids, imgids=imgids, data=data))
4887                                                                     4887
4888        # We are expecting an empty DataFrame matching types of df   4888        # We are expecting an empty DataFrame matching types of df
4889        expected = df.iloc[[], :]                                    4889        expected = df.iloc[[], :]
4890        with ensure_clean_path(self.path) as path:                   4890        with ensure_clean_path(self.path) as path:
4891            df.to_hdf(path, 'df', format='table', data_columns=True) 4891            df.to_hdf(path, 'df', format='table', data_columns=True)
4892            result = read_hdf(path, 'df', where='obsids=B')          4892            result = read_hdf(path, 'df', where='obsids=B')
4893            tm.assert_frame_equal(result, expected)                  4893            tm.assert_frame_equal(result, expected)
4894                                                                     4894
4895        # Test with categories                                       4895        # Test with categories
4896        df.obsids = df.obsids.astype('category')                     4896        df.obsids = df.obsids.astype('category')
4897        df.imgids = df.imgids.astype('category')                     4897        df.imgids = df.imgids.astype('category')
4898                                                                     4898
4899        # We are expecting an empty DataFrame matching types of df   4899        # We are expecting an empty DataFrame matching types of df
4900        expected = df.iloc[[], :]                                    4900        expected = df.iloc[[], :]
4901        with ensure_clean_path(self.path) as path:                   4901        with ensure_clean_path(self.path) as path:
4902            df.to_hdf(path, 'df', format='table', data_columns=True) 4902            df.to_hdf(path, 'df', format='table', data_columns=True)
4903            result = read_hdf(path, 'df', where='obsids=B')          4903            result = read_hdf(path, 'df', where='obsids=B')
4904            tm.assert_frame_equal(result, expected)                  4904            tm.assert_frame_equal(result, expected)
4905                                                                     4905
4906    def test_duplicate_column_name(self):                            4906    def test_duplicate_column_name(self):
4907        df = DataFrame(columns=["a", "a"], data=[[0, 0]])            4907        df = DataFrame(columns=["a", "a"], data=[[0, 0]])
4908                                                                     4908
4909        with ensure_clean_path(self.path) as path:                   4909        with ensure_clean_path(self.path) as path:
4910            pytest.raises(ValueError, df.to_hdf,                     4910            pytest.raises(ValueError, df.to_hdf,
4911                          path, 'df', format='fixed')                4911                          path, 'df', format='fixed')
4912                                                                     4912
4913            df.to_hdf(path, 'df', format='table')                    4913            df.to_hdf(path, 'df', format='table')
4914            other = read_hdf(path, 'df')                             4914            other = read_hdf(path, 'df')
4915                                                                     4915
4916            tm.assert_frame_equal(df, other)                         4916            tm.assert_frame_equal(df, other)
4917            assert df.equals(other)                                  4917            assert df.equals(other)
4918            assert other.equals(df)                                  4918            assert other.equals(df)
4919                                                                     4919
4920    def test_round_trip_equals(self):                                4920    def test_round_trip_equals(self):
4921        # GH 9330                                                    4921        # GH 9330
4922        df = DataFrame({"B": [1, 2], "A": ["x", "y"]})               4922        df = DataFrame({"B": [1, 2], "A": ["x", "y"]})
4923                                                                     4923
4924        with ensure_clean_path(self.path) as path:                   4924        with ensure_clean_path(self.path) as path:
4925            df.to_hdf(path, 'df', format='table')                    4925            df.to_hdf(path, 'df', format='table')
4926            other = read_hdf(path, 'df')                             4926            other = read_hdf(path, 'df')
4927            tm.assert_frame_equal(df, other)                         4927            tm.assert_frame_equal(df, other)
4928            assert df.equals(other)                                  4928            assert df.equals(other)
4929            assert other.equals(df)                                  4929            assert other.equals(df)
4930                                                                     4930
4931    def test_preserve_timedeltaindex_type(self):                     4931    def test_preserve_timedeltaindex_type(self):
4932        # GH9635                                                     4932        # GH9635
4933        # Storing TimedeltaIndexed DataFrames in fixed stores did no4933        # Storing TimedeltaIndexed DataFrames in fixed stores did no
                                                              t preserve                                                              t preserve
4934        # the type of the index.                                     4934        # the type of the index.
4935        df = DataFrame(np.random.normal(size=(10, 5)))               4935        df = DataFrame(np.random.normal(size=(10, 5)))
4936        df.index = timedelta_range(                                  4936        df.index = timedelta_range(
4937            start='0s', periods=10, freq='1s', name='example')       4937            start='0s', periods=10, freq='1s', name='example')
4938                                                                     4938
4939        with ensure_clean_store(self.path) as store:                 4939        with ensure_clean_store(self.path) as store:
4940                                                                     4940
4941            store['df'] = df                                         4941            store['df'] = df
4942            assert_frame_equal(store['df'], df)                      4942            assert_frame_equal(store['df'], df)
4943                                                                     4943
4944    def test_colums_multiindex_modified(self):                       4944    def test_colums_multiindex_modified(self):
4945        # BUG: 7212                                                  4945        # BUG: 7212
4946        # read_hdf store.select modified the passed columns paramete4946        # read_hdf store.select modified the passed columns paramete
                                                                      rs                                                                      rs
4947        # when multi-indexed.                                        4947        # when multi-indexed.
4948                                                                     4948
4949        df = DataFrame(np.random.rand(4, 5),                         4949        df = DataFrame(np.random.rand(4, 5),
4950                       index=list('abcd'),                           4950                       index=list('abcd'),
4951                       columns=list('ABCDE'))                        4951                       columns=list('ABCDE'))
4952        df.index.name = 'letters'                                    4952        df.index.name = 'letters'
4953        df = df.set_index(keys='E', append=True)                     4953        df = df.set_index(keys='E', append=True)
4954                                                                     4954
4955        data_columns = df.index.names + df.columns.tolist()          4955        data_columns = df.index.names + df.columns.tolist()
4956        with ensure_clean_path(self.path) as path:                   4956        with ensure_clean_path(self.path) as path:
4957            df.to_hdf(path, 'df',                                    4957            df.to_hdf(path, 'df',
4958                      mode='a',                                      4958                      mode='a',
4959                      append=True,                                   4959                      append=True,
4960                      data_columns=data_columns,                     4960                      data_columns=data_columns,
4961                      index=False)                                   4961                      index=False)
4962            cols2load = list('BCD')                                  4962            cols2load = list('BCD')
4963            cols2load_original = list(cols2load)                     4963            cols2load_original = list(cols2load)
4964            df_loaded = read_hdf(path, 'df', columns=cols2load)  # n4964            df_loaded = read_hdf(path, 'df', columns=cols2load)  # n
                                                                     oqa                                                                     oqa
4965            assert cols2load_original == cols2load                   4965            assert cols2load_original == cols2load
4966                                                                     4966
4967    def test_to_hdf_with_object_column_names(self):                  4967    def test_to_hdf_with_object_column_names(self):
4968        # GH9057                                                     4968        # GH9057
4969        # Writing HDF5 table format should only work for string-like 4969        # Writing HDF5 table format should only work for string-like
4970        # column types                                               4970        # column types
4971                                                                     4971
4972        types_should_fail = [tm.makeIntIndex, tm.makeFloatIndex,     4972        types_should_fail = [tm.makeIntIndex, tm.makeFloatIndex,
4973                             tm.makeDateIndex, tm.makeTimedeltaIndex,4973                             tm.makeDateIndex, tm.makeTimedeltaIndex,
4974                             tm.makePeriodIndex]                     4974                             tm.makePeriodIndex]
4975        types_should_run = [tm.makeStringIndex, tm.makeCategoricalIn4975        types_should_run = [tm.makeStringIndex, tm.makeCategoricalIn
                                                                    dex]                                                                    dex]
4976                                                                     4976
4977        if compat.PY3:                                               4977        if compat.PY3:
4978            types_should_run.append(tm.makeUnicodeIndex)             4978            types_should_run.append(tm.makeUnicodeIndex)
4979        else:                                                        4979        else:
4980            types_should_fail.append(tm.makeUnicodeIndex)            4980            types_should_fail.append(tm.makeUnicodeIndex)
4981                                                                     4981
4982        for index in types_should_fail:                              4982        for index in types_should_fail:
4983            df = DataFrame(np.random.randn(10, 2), columns=index(2)) 4983            df = DataFrame(np.random.randn(10, 2), columns=index(2))
4984            with ensure_clean_path(self.path) as path:               4984            with ensure_clean_path(self.path) as path:
4985                with catch_warnings(record=True):                    4985                with catch_warnings(record=True):
4986                    with pytest.raises(                              4986                    with pytest.raises(
4987                        ValueError, msg=("cannot have non-object lab4987                        ValueError, msg=("cannot have non-object lab
                                                                    el "                                                                    el "
4988                                         "DataIndexableCol")):       4988                                         "DataIndexableCol")):
4989                        df.to_hdf(path, 'df', format='table',        4989                        df.to_hdf(path, 'df', format='table',
4990                                  data_columns=True)                 4990                                  data_columns=True)
4991                                                                     4991
4992        for index in types_should_run:                               4992        for index in types_should_run:
4993            df = DataFrame(np.random.randn(10, 2), columns=index(2)) 4993            df = DataFrame(np.random.randn(10, 2), columns=index(2))
4994            with ensure_clean_path(self.path) as path:               4994            with ensure_clean_path(self.path) as path:
4995                with catch_warnings(record=True):                    4995                with catch_warnings(record=True):
4996                    df.to_hdf(path, 'df', format='table', data_colum4996                    df.to_hdf(path, 'df', format='table', data_colum
                                                                ns=True)                                                                ns=True)
4997                    result = pd.read_hdf(                            4997                    result = pd.read_hdf(
4998                        path, 'df', where="index = [{0}]".format(df.4998                        path, 'df', where="index = [{0}]".format(df.
                                                              index[0]))                                                              index[0]))
4999                    assert(len(result))                              4999                    assert(len(result))
5000                                                                     5000
5001    def test_read_hdf_open_store(self):                              5001    def test_read_hdf_open_store(self):
5002        # GH10330                                                    5002        # GH10330
5003        # No check for non-string path_or-buf, and no test of open s5003        # No check for non-string path_or-buf, and no test of open s
                                                                    tore                                                                    tore
5004        df = DataFrame(np.random.rand(4, 5),                         5004        df = DataFrame(np.random.rand(4, 5),
5005                       index=list('abcd'),                           5005                       index=list('abcd'),
5006                       columns=list('ABCDE'))                        5006                       columns=list('ABCDE'))
5007        df.index.name = 'letters'                                    5007        df.index.name = 'letters'
5008        df = df.set_index(keys='E', append=True)                     5008        df = df.set_index(keys='E', append=True)
5009                                                                     5009
5010        with ensure_clean_path(self.path) as path:                   5010        with ensure_clean_path(self.path) as path:
5011            df.to_hdf(path, 'df', mode='w')                          5011            df.to_hdf(path, 'df', mode='w')
5012            direct = read_hdf(path, 'df')                            5012            direct = read_hdf(path, 'df')
5013            store = HDFStore(path, mode='r')                         5013            store = HDFStore(path, mode='r')
5014            indirect = read_hdf(store, 'df')                         5014            indirect = read_hdf(store, 'df')
5015            tm.assert_frame_equal(direct, indirect)                  5015            tm.assert_frame_equal(direct, indirect)
5016            assert store.is_open                                     5016            assert store.is_open
5017            store.close()                                            5017            store.close()
5018                                                                     5018
5019    def test_read_hdf_iterator(self):                                5019    def test_read_hdf_iterator(self):
5020        df = DataFrame(np.random.rand(4, 5),                         5020        df = DataFrame(np.random.rand(4, 5),
5021                       index=list('abcd'),                           5021                       index=list('abcd'),
5022                       columns=list('ABCDE'))                        5022                       columns=list('ABCDE'))
5023        df.index.name = 'letters'                                    5023        df.index.name = 'letters'
5024        df = df.set_index(keys='E', append=True)                     5024        df = df.set_index(keys='E', append=True)
5025                                                                     5025
5026        with ensure_clean_path(self.path) as path:                   5026        with ensure_clean_path(self.path) as path:
5027            df.to_hdf(path, 'df', mode='w', format='t')              5027            df.to_hdf(path, 'df', mode='w', format='t')
5028            direct = read_hdf(path, 'df')                            5028            direct = read_hdf(path, 'df')
5029            iterator = read_hdf(path, 'df', iterator=True)           5029            iterator = read_hdf(path, 'df', iterator=True)
5030            assert isinstance(iterator, TableIterator)               5030            assert isinstance(iterator, TableIterator)
5031            indirect = next(iterator.__iter__())                     5031            indirect = next(iterator.__iter__())
5032            tm.assert_frame_equal(direct, indirect)                  5032            tm.assert_frame_equal(direct, indirect)
5033            iterator.store.close()                                   5033            iterator.store.close()
5034                                                                     5034
5035    def test_read_hdf_errors(self):                                  5035    def test_read_hdf_errors(self):
5036        df = DataFrame(np.random.rand(4, 5),                         5036        df = DataFrame(np.random.rand(4, 5),
5037                       index=list('abcd'),                           5037                       index=list('abcd'),
5038                       columns=list('ABCDE'))                        5038                       columns=list('ABCDE'))
5039                                                                     5039
5040        with ensure_clean_path(self.path) as path:                   5040        with ensure_clean_path(self.path) as path:
5041            pytest.raises(IOError, read_hdf, path, 'key')            5041            pytest.raises(IOError, read_hdf, path, 'key')
5042            df.to_hdf(path, 'df')                                    5042            df.to_hdf(path, 'df')
5043            store = HDFStore(path, mode='r')                         5043            store = HDFStore(path, mode='r')
5044            store.close()                                            5044            store.close()
5045            pytest.raises(IOError, read_hdf, store, 'df')            5045            pytest.raises(IOError, read_hdf, store, 'df')
5046            with open(path, mode='r') as store:                      5046            with open(path, mode='r') as store:
5047                pytest.raises(NotImplementedError, read_hdf, store, 5047                pytest.raises(NotImplementedError, read_hdf, store, 
                                                                   'df')                                                                   'df')
5048                                                                     5048
5049    def test_invalid_complib(self):                                  5049    def test_invalid_complib(self):
5050        df = DataFrame(np.random.rand(4, 5),                         5050        df = DataFrame(np.random.rand(4, 5),
5051                       index=list('abcd'),                           5051                       index=list('abcd'),
5052                       columns=list('ABCDE'))                        5052                       columns=list('ABCDE'))
5053        with ensure_clean_path(self.path) as path:                   5053        with ensure_clean_path(self.path) as path:
5054            with pytest.raises(ValueError):                          5054            with pytest.raises(ValueError):
5055                df.to_hdf(path, 'df', complib='foolib')              5055                df.to_hdf(path, 'df', complib='foolib')
5056    # GH10443                                                        5056    # GH10443
5057                                                                     5057
5058    def test_read_nokey(self):                                       5058    def test_read_nokey(self):
5059        df = DataFrame(np.random.rand(4, 5),                         5059        df = DataFrame(np.random.rand(4, 5),
5060                       index=list('abcd'),                           5060                       index=list('abcd'),
5061                       columns=list('ABCDE'))                        5061                       columns=list('ABCDE'))
5062                                                                     5062
5063        # Categorical dtype not supported for "fixed" format. So no 5063        # Categorical dtype not supported for "fixed" format. So no 
                                                                    need                                                                    need
5064        # to test with that dtype in the dataframe here.             5064        # to test with that dtype in the dataframe here.
5065        with ensure_clean_path(self.path) as path:                   5065        with ensure_clean_path(self.path) as path:
5066            df.to_hdf(path, 'df', mode='a')                          5066            df.to_hdf(path, 'df', mode='a')
5067            reread = read_hdf(path)                                  5067            reread = read_hdf(path)
5068            assert_frame_equal(df, reread)                           5068            assert_frame_equal(df, reread)
5069            df.to_hdf(path, 'df2', mode='a')                         5069            df.to_hdf(path, 'df2', mode='a')
5070            pytest.raises(ValueError, read_hdf, path)                5070            pytest.raises(ValueError, read_hdf, path)
5071                                                                     5071
5072    def test_read_nokey_table(self):                                 5072    def test_read_nokey_table(self):
5073        # GH13231                                                    5073        # GH13231
5074        df = DataFrame({'i': range(5),                               5074        df = DataFrame({'i': range(5),
5075                        'c': Series(list('abacd'), dtype='category')5075                        'c': Series(list('abacd'), dtype='category')
                                                                      })                                                                      })
5076                                                                     5076
5077        with ensure_clean_path(self.path) as path:                   5077        with ensure_clean_path(self.path) as path:
5078            df.to_hdf(path, 'df', mode='a', format='table')          5078            df.to_hdf(path, 'df', mode='a', format='table')
5079            reread = read_hdf(path)                                  5079            reread = read_hdf(path)
5080            assert_frame_equal(df, reread)                           5080            assert_frame_equal(df, reread)
5081            df.to_hdf(path, 'df2', mode='a', format='table')         5081            df.to_hdf(path, 'df2', mode='a', format='table')
5082            pytest.raises(ValueError, read_hdf, path)                5082            pytest.raises(ValueError, read_hdf, path)
5083                                                                     5083
5084    def test_read_nokey_empty(self):                                 5084    def test_read_nokey_empty(self):
5085        with ensure_clean_path(self.path) as path:                   5085        with ensure_clean_path(self.path) as path:
5086            store = HDFStore(path)                                   5086            store = HDFStore(path)
5087            store.close()                                            5087            store.close()
5088            pytest.raises(ValueError, read_hdf, path)                5088            pytest.raises(ValueError, read_hdf, path)
5089                                                                     5089
5090    def test_read_from_pathlib_path(self):                           5090    def test_read_from_pathlib_path(self):
5091                                                                     5091
5092        # GH11773                                                    5092        # GH11773
5093        tm._skip_if_no_pathlib()                                     5093        tm._skip_if_no_pathlib()
5094                                                                     5094
5095        from pathlib import Path                                     5095        from pathlib import Path
5096                                                                     5096
5097        expected = DataFrame(np.random.rand(4, 5),                   5097        expected = DataFrame(np.random.rand(4, 5),
5098                             index=list('abcd'),                     5098                             index=list('abcd'),
5099                             columns=list('ABCDE'))                  5099                             columns=list('ABCDE'))
5100        with ensure_clean_path(self.path) as filename:               5100        with ensure_clean_path(self.path) as filename:
5101            path_obj = Path(filename)                                5101            path_obj = Path(filename)
5102                                                                     5102
5103            expected.to_hdf(path_obj, 'df', mode='a')                5103            expected.to_hdf(path_obj, 'df', mode='a')
5104            actual = read_hdf(path_obj, 'df')                        5104            actual = read_hdf(path_obj, 'df')
5105                                                                     5105
5106        tm.assert_frame_equal(expected, actual)                      5106        tm.assert_frame_equal(expected, actual)
5107                                                                     5107
5108    def test_read_from_py_localpath(self):                           5108    def test_read_from_py_localpath(self):
5109                                                                     5109
5110        # GH11773                                                    5110        # GH11773
5111        tm._skip_if_no_localpath()                                   5111        tm._skip_if_no_localpath()
5112                                                                     5112
5113        from py.path import local as LocalPath                       5113        from py.path import local as LocalPath
5114                                                                     5114
5115        expected = DataFrame(np.random.rand(4, 5),                   5115        expected = DataFrame(np.random.rand(4, 5),
5116                             index=list('abcd'),                     5116                             index=list('abcd'),
5117                             columns=list('ABCDE'))                  5117                             columns=list('ABCDE'))
5118        with ensure_clean_path(self.path) as filename:               5118        with ensure_clean_path(self.path) as filename:
5119            path_obj = LocalPath(filename)                           5119            path_obj = LocalPath(filename)
5120                                                                     5120
5121            expected.to_hdf(path_obj, 'df', mode='a')                5121            expected.to_hdf(path_obj, 'df', mode='a')
5122            actual = read_hdf(path_obj, 'df')                        5122            actual = read_hdf(path_obj, 'df')
5123                                                                     5123
5124        tm.assert_frame_equal(expected, actual)                      5124        tm.assert_frame_equal(expected, actual)
5125                                                                     5125
5126    def test_query_long_float_literal(self):                         5126    def test_query_long_float_literal(self):
5127        # GH 14241                                                   5127        # GH 14241
5128        df = pd.DataFrame({'A': [1000000000.0009,                    5128        df = pd.DataFrame({'A': [1000000000.0009,
5129                                 1000000000.0011,                    5129                                 1000000000.0011,
5130                                 1000000000.0015]})                  5130                                 1000000000.0015]})
5131                                                                     5131
5132        with ensure_clean_store(self.path) as store:                 5132        with ensure_clean_store(self.path) as store:
5133            store.append('test', df, format='table', data_columns=Tr5133            store.append('test', df, format='table', data_columns=Tr
                                                                     ue)                                                                     ue)
5134                                                                     5134
5135            cutoff = 1000000000.0006                                 5135            cutoff = 1000000000.0006
5136            result = store.select('test', "A < %.4f" % cutoff)       5136            result = store.select('test', "A < %.4f" % cutoff)
5137            assert result.empty                                      5137            assert result.empty
5138                                                                     5138
5139            cutoff = 1000000000.0010                                 5139            cutoff = 1000000000.0010
5140            result = store.select('test', "A > %.4f" % cutoff)       5140            result = store.select('test', "A > %.4f" % cutoff)
5141            expected = df.loc[[1, 2], :]                             5141            expected = df.loc[[1, 2], :]
5142            tm.assert_frame_equal(expected, result)                  5142            tm.assert_frame_equal(expected, result)
5143                                                                     5143
5144            exact = 1000000000.0011                                  5144            exact = 1000000000.0011
5145            result = store.select('test', 'A == %.4f' % exact)       5145            result = store.select('test', 'A == %.4f' % exact)
5146            expected = df.loc[[1], :]                                5146            expected = df.loc[[1], :]
5147            tm.assert_frame_equal(expected, result)                  5147            tm.assert_frame_equal(expected, result)
5148                                                                     5148
5149    def test_query_compare_column_type(self):                        5149    def test_query_compare_column_type(self):
5150        # GH 15492                                                   5150        # GH 15492
5151        df = pd.DataFrame({'date': ['2014-01-01', '2014-01-02'],     5151        df = pd.DataFrame({'date': ['2014-01-01', '2014-01-02'],
5152                           'real_date': date_range('2014-01-01', per5152                           'real_date': date_range('2014-01-01', per
                                                                iods=2),                                                                iods=2),
5153                           'float': [1.1, 1.2],                      5153                           'float': [1.1, 1.2],
5154                           'int': [1, 2]},                           5154                           'int': [1, 2]},
5155                          columns=['date', 'real_date', 'float', 'in5155                          columns=['date', 'real_date', 'float', 'in
                                                                    t'])                                                                    t'])
5156                                                                     5156
5157        with ensure_clean_store(self.path) as store:                 5157        with ensure_clean_store(self.path) as store:
5158            store.append('test', df, format='table', data_columns=Tr5158            store.append('test', df, format='table', data_columns=Tr
                                                                     ue)                                                                     ue)
5159                                                                     5159
5160            ts = pd.Timestamp('2014-01-01')  # noqa                  5160            ts = pd.Timestamp('2014-01-01')  # noqa
5161            result = store.select('test', where='real_date > ts')    5161            result = store.select('test', where='real_date > ts')
5162            expected = df.loc[[1], :]                                5162            expected = df.loc[[1], :]
5163            tm.assert_frame_equal(expected, result)                  5163            tm.assert_frame_equal(expected, result)
5164                                                                     5164
5165            for op in ['<', '>', '==']:                              5165            for op in ['<', '>', '==']:
5166                # non strings to string column always fail           5166                # non strings to string column always fail
5167                for v in [2.1, True, pd.Timestamp('2014-01-01'),     5167                for v in [2.1, True, pd.Timestamp('2014-01-01'),
5168                          pd.Timedelta(1, 's')]:                     5168                          pd.Timedelta(1, 's')]:
5169                    query = 'date {op} v'.format(op=op)              5169                    query = 'date {op} v'.format(op=op)
5170                    with pytest.raises(TypeError):                   5170                    with pytest.raises(TypeError):
5171                        result = store.select('test', where=query)   5171                        result = store.select('test', where=query)
5172                                                                     5172
5173                # strings to other columns must be convertible to ty5173                # strings to other columns must be convertible to ty
                                                                      pe                                                                      pe
5174                v = 'a'                                              5174                v = 'a'
5175                for col in ['int', 'float', 'real_date']:            5175                for col in ['int', 'float', 'real_date']:
5176                    query = '{col} {op} v'.format(op=op, col=col)    5176                    query = '{col} {op} v'.format(op=op, col=col)
5177                    with pytest.raises(ValueError):                  5177                    with pytest.raises(ValueError):
5178                        result = store.select('test', where=query)   5178                        result = store.select('test', where=query)
5179                                                                     5179
5180                for v, col in zip(['1', '1.1', '2014-01-01'],        5180                for v, col in zip(['1', '1.1', '2014-01-01'],
5181                                  ['int', 'float', 'real_date']):    5181                                  ['int', 'float', 'real_date']):
5182                    query = '{col} {op} v'.format(op=op, col=col)    5182                    query = '{col} {op} v'.format(op=op, col=col)
5183                    result = store.select('test', where=query)       5183                    result = store.select('test', where=query)
5184                                                                     5184
5185                    if op == '==':                                   5185                    if op == '==':
5186                        expected = df.loc[[0], :]                    5186                        expected = df.loc[[0], :]
5187                    elif op == '>':                                  5187                    elif op == '>':
5188                        expected = df.loc[[1], :]                    5188                        expected = df.loc[[1], :]
5189                    else:                                            5189                    else:
5190                        expected = df.loc[[], :]                     5190                        expected = df.loc[[], :]
5191                    tm.assert_frame_equal(expected, result)          5191                    tm.assert_frame_equal(expected, result)
5192                                                                     5192
5193    @pytest.mark.parametrize('format', ['fixed', 'table'])           5193    @pytest.mark.parametrize('format', ['fixed', 'table'])
5194    def test_read_hdf_series_mode_r(self, format):                   5194    def test_read_hdf_series_mode_r(self, format):
5195        # GH 16583                                                   5195        # GH 16583
5196        # Tests that reading a Series saved to an HDF file           5196        # Tests that reading a Series saved to an HDF file
5197        # still works if a mode='r' argument is supplied             5197        # still works if a mode='r' argument is supplied
5198        series = tm.makeFloatSeries()                                5198        series = tm.makeFloatSeries()
5199        with ensure_clean_path(self.path) as path:                   5199        with ensure_clean_path(self.path) as path:
5200            series.to_hdf(path, key='data', format=format)           5200            series.to_hdf(path, key='data', format=format)
5201            result = pd.read_hdf(path, key='data', mode='r')         5201            result = pd.read_hdf(path, key='data', mode='r')
5202        tm.assert_series_equal(result, series)                       5202        tm.assert_series_equal(result, series)
5203                                                                     5203
5204    def test_read_py2_hdf_file_in_py3(self):                         5204    def test_read_py2_hdf_file_in_py3(self):
5205        # GH 16781                                                   5205        # GH 16781
5206                                                                     5206
5207        # tests reading a PeriodIndex DataFrame written in Python2 i5207        # tests reading a PeriodIndex DataFrame written in Python2 i
                                                               n Python3                                                               n Python3
5208                                                                     5208
5209        # the file was generated in Python 2.7 like so:              5209        # the file was generated in Python 2.7 like so:
5210        #                                                            5210        #
5211        # df = pd.DataFrame([1.,2,3], index=pd.PeriodIndex(          5211        # df = pd.DataFrame([1.,2,3], index=pd.PeriodIndex(
5212        #              ['2015-01-01', '2015-01-02', '2015-01-05'], f5212        #              ['2015-01-01', '2015-01-02', '2015-01-05'], f
                                                               req='B'))                                                               req='B'))
5213        # df.to_hdf('periodindex_0.20.1_x86_64_darwin_2.7.13.h5', 'p5213        # df.to_hdf('periodindex_0.20.1_x86_64_darwin_2.7.13.h5', 'p
                                                                      ')                                                                      ')
5214                                                                     5214
5215        expected = pd.DataFrame([1., 2, 3], index=pd.PeriodIndex(    5215        expected = pd.DataFrame([1., 2, 3], index=pd.PeriodIndex(
5216            ['2015-01-01', '2015-01-02', '2015-01-05'], freq='B'))   5216            ['2015-01-01', '2015-01-02', '2015-01-05'], freq='B'))
5217                                                                     5217
5218        with ensure_clean_store(                                     5218        with ensure_clean_store(
5219                tm.get_data_path('periodindex_0.20.1_x86_64_darwin_25219                tm.get_data_path(                                    
    .7.13.h5'),    
                                                                         5220                    'legacy_hdf/periodindex_0.20.1_x86_64_darwin_2.7
                                                                                                                                        .13.h5'),
5220                mode='r') as store:                                  5221                mode='r') as store:
5221            result = store['p']                                      5222            result = store['p']
5222            assert_frame_equal(result, expected)                     5223            assert_frame_equal(result, expected)
5223                                                                     5224
5224                                                                     5225
5225class TestHDFComplexValues(Base):                                    5226class TestHDFComplexValues(Base):
5226    # GH10447                                                        5227    # GH10447
5227                                                                     5228
5228    def test_complex_fixed(self):                                    5229    def test_complex_fixed(self):
5229        df = DataFrame(np.random.rand(4, 5).astype(np.complex64),    5230        df = DataFrame(np.random.rand(4, 5).astype(np.complex64),
5230                       index=list('abcd'),                           5231                       index=list('abcd'),
5231                       columns=list('ABCDE'))                        5232                       columns=list('ABCDE'))
5232                                                                     5233
5233        with ensure_clean_path(self.path) as path:                   5234        with ensure_clean_path(self.path) as path:
5234            df.to_hdf(path, 'df')                                    5235            df.to_hdf(path, 'df')
5235            reread = read_hdf(path, 'df')                            5236            reread = read_hdf(path, 'df')
5236            assert_frame_equal(df, reread)                           5237            assert_frame_equal(df, reread)
5237                                                                     5238
5238        df = DataFrame(np.random.rand(4, 5).astype(np.complex128),   5239        df = DataFrame(np.random.rand(4, 5).astype(np.complex128),
5239                       index=list('abcd'),                           5240                       index=list('abcd'),
5240                       columns=list('ABCDE'))                        5241                       columns=list('ABCDE'))
5241        with ensure_clean_path(self.path) as path:                   5242        with ensure_clean_path(self.path) as path:
5242            df.to_hdf(path, 'df')                                    5243            df.to_hdf(path, 'df')
5243            reread = read_hdf(path, 'df')                            5244            reread = read_hdf(path, 'df')
5244            assert_frame_equal(df, reread)                           5245            assert_frame_equal(df, reread)
5245                                                                     5246
5246    def test_complex_table(self):                                    5247    def test_complex_table(self):
5247        df = DataFrame(np.random.rand(4, 5).astype(np.complex64),    5248        df = DataFrame(np.random.rand(4, 5).astype(np.complex64),
5248                       index=list('abcd'),                           5249                       index=list('abcd'),
5249                       columns=list('ABCDE'))                        5250                       columns=list('ABCDE'))
5250                                                                     5251
5251        with ensure_clean_path(self.path) as path:                   5252        with ensure_clean_path(self.path) as path:
5252            df.to_hdf(path, 'df', format='table')                    5253            df.to_hdf(path, 'df', format='table')
5253            reread = read_hdf(path, 'df')                            5254            reread = read_hdf(path, 'df')
5254            assert_frame_equal(df, reread)                           5255            assert_frame_equal(df, reread)
5255                                                                     5256
5256        df = DataFrame(np.random.rand(4, 5).astype(np.complex128),   5257        df = DataFrame(np.random.rand(4, 5).astype(np.complex128),
5257                       index=list('abcd'),                           5258                       index=list('abcd'),
5258                       columns=list('ABCDE'))                        5259                       columns=list('ABCDE'))
5259                                                                     5260
5260        with ensure_clean_path(self.path) as path:                   5261        with ensure_clean_path(self.path) as path:
5261            df.to_hdf(path, 'df', format='table', mode='w')          5262            df.to_hdf(path, 'df', format='table', mode='w')
5262            reread = read_hdf(path, 'df')                            5263            reread = read_hdf(path, 'df')
5263            assert_frame_equal(df, reread)                           5264            assert_frame_equal(df, reread)
5264                                                                     5265
5265    def test_complex_mixed_fixed(self):                              5266    def test_complex_mixed_fixed(self):
5266        complex64 = np.array([1.0 + 1.0j, 1.0 + 1.0j,                5267        complex64 = np.array([1.0 + 1.0j, 1.0 + 1.0j,
5267                              1.0 + 1.0j, 1.0 + 1.0j], dtype=np.comp5268                              1.0 + 1.0j, 1.0 + 1.0j], dtype=np.comp
                                                                  lex64)                                                                  lex64)
5268        complex128 = np.array([1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 15269        complex128 = np.array([1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1
                                                             .0 + 1.0j],                                                             .0 + 1.0j],
5269                              dtype=np.complex128)                   5270                              dtype=np.complex128)
5270        df = DataFrame({'A': [1, 2, 3, 4],                           5271        df = DataFrame({'A': [1, 2, 3, 4],
5271                        'B': ['a', 'b', 'c', 'd'],                   5272                        'B': ['a', 'b', 'c', 'd'],
5272                        'C': complex64,                              5273                        'C': complex64,
5273                        'D': complex128,                             5274                        'D': complex128,
5274                        'E': [1.0, 2.0, 3.0, 4.0]},                  5275                        'E': [1.0, 2.0, 3.0, 4.0]},
5275                       index=list('abcd'))                           5276                       index=list('abcd'))
5276        with ensure_clean_path(self.path) as path:                   5277        with ensure_clean_path(self.path) as path:
5277            df.to_hdf(path, 'df')                                    5278            df.to_hdf(path, 'df')
5278            reread = read_hdf(path, 'df')                            5279            reread = read_hdf(path, 'df')
5279            assert_frame_equal(df, reread)                           5280            assert_frame_equal(df, reread)
5280                                                                     5281
5281    def test_complex_mixed_table(self):                              5282    def test_complex_mixed_table(self):
5282        complex64 = np.array([1.0 + 1.0j, 1.0 + 1.0j,                5283        complex64 = np.array([1.0 + 1.0j, 1.0 + 1.0j,
5283                              1.0 + 1.0j, 1.0 + 1.0j], dtype=np.comp5284                              1.0 + 1.0j, 1.0 + 1.0j], dtype=np.comp
                                                                  lex64)                                                                  lex64)
5284        complex128 = np.array([1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 15285        complex128 = np.array([1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1
                                                             .0 + 1.0j],                                                             .0 + 1.0j],
5285                              dtype=np.complex128)                   5286                              dtype=np.complex128)
5286        df = DataFrame({'A': [1, 2, 3, 4],                           5287        df = DataFrame({'A': [1, 2, 3, 4],
5287                        'B': ['a', 'b', 'c', 'd'],                   5288                        'B': ['a', 'b', 'c', 'd'],
5288                        'C': complex64,                              5289                        'C': complex64,
5289                        'D': complex128,                             5290                        'D': complex128,
5290                        'E': [1.0, 2.0, 3.0, 4.0]},                  5291                        'E': [1.0, 2.0, 3.0, 4.0]},
5291                       index=list('abcd'))                           5292                       index=list('abcd'))
5292                                                                     5293
5293        with ensure_clean_store(self.path) as store:                 5294        with ensure_clean_store(self.path) as store:
5294            store.append('df', df, data_columns=['A', 'B'])          5295            store.append('df', df, data_columns=['A', 'B'])
5295            result = store.select('df', where='A>2')                 5296            result = store.select('df', where='A>2')
5296            assert_frame_equal(df.loc[df.A > 2], result)             5297            assert_frame_equal(df.loc[df.A > 2], result)
5297                                                                     5298
5298        with ensure_clean_path(self.path) as path:                   5299        with ensure_clean_path(self.path) as path:
5299            df.to_hdf(path, 'df', format='table')                    5300            df.to_hdf(path, 'df', format='table')
5300            reread = read_hdf(path, 'df')                            5301            reread = read_hdf(path, 'df')
5301            assert_frame_equal(df, reread)                           5302            assert_frame_equal(df, reread)
5302                                                                     5303
5303    def test_complex_across_dimensions_fixed(self):                  5304    def test_complex_across_dimensions_fixed(self):
5304        with catch_warnings(record=True):                            5305        with catch_warnings(record=True):
5305            complex128 = np.array(                                   5306            complex128 = np.array(
5306                [1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j])    5307                [1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j])
5307            s = Series(complex128, index=list('abcd'))               5308            s = Series(complex128, index=list('abcd'))
5308            df = DataFrame({'A': s, 'B': s})                         5309            df = DataFrame({'A': s, 'B': s})
5309            p = Panel({'One': df, 'Two': df})                        5310            p = Panel({'One': df, 'Two': df})
5310                                                                     5311
5311            objs = [s, df, p]                                        5312            objs = [s, df, p]
5312            comps = [tm.assert_series_equal, tm.assert_frame_equal,  5313            comps = [tm.assert_series_equal, tm.assert_frame_equal,
5313                     tm.assert_panel_equal]                          5314                     tm.assert_panel_equal]
5314            for obj, comp in zip(objs, comps):                       5315            for obj, comp in zip(objs, comps):
5315                with ensure_clean_path(self.path) as path:           5316                with ensure_clean_path(self.path) as path:
5316                    obj.to_hdf(path, 'obj', format='fixed')          5317                    obj.to_hdf(path, 'obj', format='fixed')
5317                    reread = read_hdf(path, 'obj')                   5318                    reread = read_hdf(path, 'obj')
5318                    comp(obj, reread)                                5319                    comp(obj, reread)
5319                                                                     5320
5320    def test_complex_across_dimensions(self):                        5321    def test_complex_across_dimensions(self):
5321        complex128 = np.array([1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 15322        complex128 = np.array([1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1
                                                             .0 + 1.0j])                                                             .0 + 1.0j])
5322        s = Series(complex128, index=list('abcd'))                   5323        s = Series(complex128, index=list('abcd'))
5323        df = DataFrame({'A': s, 'B': s})                             5324        df = DataFrame({'A': s, 'B': s})
5324                                                                     5325
5325        with catch_warnings(record=True):                            5326        with catch_warnings(record=True):
5326            p = Panel({'One': df, 'Two': df})                        5327            p = Panel({'One': df, 'Two': df})
5327            p4d = Panel4D({'i': p, 'ii': p})                         5328            p4d = Panel4D({'i': p, 'ii': p})
5328                                                                     5329
5329            objs = [df, p, p4d]                                      5330            objs = [df, p, p4d]
5330            comps = [tm.assert_frame_equal, tm.assert_panel_equal,   5331            comps = [tm.assert_frame_equal, tm.assert_panel_equal,
5331                     tm.assert_panel4d_equal]                        5332                     tm.assert_panel4d_equal]
5332            for obj, comp in zip(objs, comps):                       5333            for obj, comp in zip(objs, comps):
5333                with ensure_clean_path(self.path) as path:           5334                with ensure_clean_path(self.path) as path:
5334                    obj.to_hdf(path, 'obj', format='table')          5335                    obj.to_hdf(path, 'obj', format='table')
5335                    reread = read_hdf(path, 'obj')                   5336                    reread = read_hdf(path, 'obj')
5336                    comp(obj, reread)                                5337                    comp(obj, reread)
5337                                                                     5338
5338    def test_complex_indexing_error(self):                           5339    def test_complex_indexing_error(self):
5339        complex128 = np.array([1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 15340        complex128 = np.array([1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1
                                                             .0 + 1.0j],                                                             .0 + 1.0j],
5340                              dtype=np.complex128)                   5341                              dtype=np.complex128)
5341        df = DataFrame({'A': [1, 2, 3, 4],                           5342        df = DataFrame({'A': [1, 2, 3, 4],
5342                        'B': ['a', 'b', 'c', 'd'],                   5343                        'B': ['a', 'b', 'c', 'd'],
5343                        'C': complex128},                            5344                        'C': complex128},
5344                       index=list('abcd'))                           5345                       index=list('abcd'))
5345        with ensure_clean_store(self.path) as store:                 5346        with ensure_clean_store(self.path) as store:
5346            pytest.raises(TypeError, store.append,                   5347            pytest.raises(TypeError, store.append,
5347                          'df', df, data_columns=['C'])              5348                          'df', df, data_columns=['C'])
5348                                                                     5349
5349    def test_complex_series_error(self):                             5350    def test_complex_series_error(self):
5350        complex128 = np.array([1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 15351        complex128 = np.array([1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1
                                                             .0 + 1.0j])                                                             .0 + 1.0j])
5351        s = Series(complex128, index=list('abcd'))                   5352        s = Series(complex128, index=list('abcd'))
5352                                                                     5353
5353        with ensure_clean_path(self.path) as path:                   5354        with ensure_clean_path(self.path) as path:
5354            pytest.raises(TypeError, s.to_hdf, path, 'obj', format='5355            pytest.raises(TypeError, s.to_hdf, path, 'obj', format='
                                                                     t')                                                                     t')
5355                                                                     5356
5356        with ensure_clean_path(self.path) as path:                   5357        with ensure_clean_path(self.path) as path:
5357            s.to_hdf(path, 'obj', format='t', index=False)           5358            s.to_hdf(path, 'obj', format='t', index=False)
5358            reread = read_hdf(path, 'obj')                           5359            reread = read_hdf(path, 'obj')
5359            tm.assert_series_equal(s, reread)                        5360            tm.assert_series_equal(s, reread)
5360                                                                     5361
5361    def test_complex_append(self):                                   5362    def test_complex_append(self):
5362        df = DataFrame({'a': np.random.randn(100).astype(np.complex15363        df = DataFrame({'a': np.random.randn(100).astype(np.complex1
                                                                    28),                                                                    28),
5363                        'b': np.random.randn(100)})                  5364                        'b': np.random.randn(100)})
5364                                                                     5365
5365        with ensure_clean_store(self.path) as store:                 5366        with ensure_clean_store(self.path) as store:
5366            store.append('df', df, data_columns=['b'])               5367            store.append('df', df, data_columns=['b'])
5367            store.append('df', df)                                   5368            store.append('df', df)
5368            result = store.select('df')                              5369            result = store.select('df')
5369            assert_frame_equal(pd.concat([df, df], 0), result)       5370            assert_frame_equal(pd.concat([df, df], 0), result)
5370                                                                     5371
5371                                                                     5372
5372class TestTimezones(Base):                                           5373class TestTimezones(Base):
5373                                                                     5374
5374    def _compare_with_tz(self, a, b):                                5375    def _compare_with_tz(self, a, b):
5375        tm.assert_frame_equal(a, b)                                  5376        tm.assert_frame_equal(a, b)
5376                                                                     5377
5377        # compare the zones on each element                          5378        # compare the zones on each element
5378        for c in a.columns:                                          5379        for c in a.columns:
5379            for i in a.index:                                        5380            for i in a.index:
5380                a_e = a.loc[i, c]                                    5381                a_e = a.loc[i, c]
5381                b_e = b.loc[i, c]                                    5382                b_e = b.loc[i, c]
5382                if not (a_e == b_e and a_e.tz == b_e.tz):            5383                if not (a_e == b_e and a_e.tz == b_e.tz):
5383                    raise AssertionError(                            5384                    raise AssertionError(
5384                        "invalid tz comparsion [%s] [%s]" % (a_e, b_5385                        "invalid tz comparsion [%s] [%s]" % (a_e, b_
                                                                     e))                                                                     e))
5385                                                                     5386
5386    def test_append_with_timezones_dateutil(self):                   5387    def test_append_with_timezones_dateutil(self):
5387                                                                     5388
5388        from datetime import timedelta                               5389        from datetime import timedelta
5389        tm._skip_if_no_dateutil()                                    5390        tm._skip_if_no_dateutil()
5390                                                                     5391
5391        # use maybe_get_tz instead of dateutil.tz.gettz to handle th5392        # use maybe_get_tz instead of dateutil.tz.gettz to handle th
                                                               e windows                                                               e windows
5392        # filename issues.                                           5393        # filename issues.
5393        from pandas._libs.tslib import maybe_get_tz                  5394        from pandas._libs.tslib import maybe_get_tz
5394        gettz = lambda x: maybe_get_tz('dateutil/' + x)              5395        gettz = lambda x: maybe_get_tz('dateutil/' + x)
5395                                                                     5396
5396        # as columns                                                 5397        # as columns
5397        with ensure_clean_store(self.path) as store:                 5398        with ensure_clean_store(self.path) as store:
5398                                                                     5399
5399            _maybe_remove(store, 'df_tz')                            5400            _maybe_remove(store, 'df_tz')
5400            df = DataFrame(dict(A=[Timestamp('20130102 2:00:00', tz=5401            df = DataFrame(dict(A=[Timestamp('20130102 2:00:00', tz=
                                                                  gettz(                                                                  gettz(
5401                'US/Eastern')) + timedelta(hours=1) * i for i in ran5402                'US/Eastern')) + timedelta(hours=1) * i for i in ran
                                                                ge(5)]))                                                                ge(5)]))
5402                                                                     5403
5403            store.append('df_tz', df, data_columns=['A'])            5404            store.append('df_tz', df, data_columns=['A'])
5404            result = store['df_tz']                                  5405            result = store['df_tz']
5405            self._compare_with_tz(result, df)                        5406            self._compare_with_tz(result, df)
5406            assert_frame_equal(result, df)                           5407            assert_frame_equal(result, df)
5407                                                                     5408
5408            # select with tz aware                                   5409            # select with tz aware
5409            expected = df[df.A >= df.A[3]]                           5410            expected = df[df.A >= df.A[3]]
5410            result = store.select('df_tz', where='A>=df.A[3]')       5411            result = store.select('df_tz', where='A>=df.A[3]')
5411            self._compare_with_tz(result, expected)                  5412            self._compare_with_tz(result, expected)
5412                                                                     5413
5413            # ensure we include dates in DST and STD time here.      5414            # ensure we include dates in DST and STD time here.
5414            _maybe_remove(store, 'df_tz')                            5415            _maybe_remove(store, 'df_tz')
5415            df = DataFrame(dict(A=Timestamp('20130102',              5416            df = DataFrame(dict(A=Timestamp('20130102',
5416                                            tz=gettz('US/Eastern')), 5417                                            tz=gettz('US/Eastern')),
5417                                B=Timestamp('20130603',              5418                                B=Timestamp('20130603',
5418                                            tz=gettz('US/Eastern'))),5419                                            tz=gettz('US/Eastern'))),
5419                           index=range(5))                           5420                           index=range(5))
5420            store.append('df_tz', df)                                5421            store.append('df_tz', df)
5421            result = store['df_tz']                                  5422            result = store['df_tz']
5422            self._compare_with_tz(result, df)                        5423            self._compare_with_tz(result, df)
5423            assert_frame_equal(result, df)                           5424            assert_frame_equal(result, df)
5424                                                                     5425
5425            df = DataFrame(dict(A=Timestamp('20130102',              5426            df = DataFrame(dict(A=Timestamp('20130102',
5426                                            tz=gettz('US/Eastern')), 5427                                            tz=gettz('US/Eastern')),
5427                                B=Timestamp('20130102', tz=gettz('EE5428                                B=Timestamp('20130102', tz=gettz('EE
                                                                  T'))),                                                                  T'))),
5428                           index=range(5))                           5429                           index=range(5))
5429            pytest.raises(ValueError, store.append, 'df_tz', df)     5430            pytest.raises(ValueError, store.append, 'df_tz', df)
5430                                                                     5431
5431            # this is ok                                             5432            # this is ok
5432            _maybe_remove(store, 'df_tz')                            5433            _maybe_remove(store, 'df_tz')
5433            store.append('df_tz', df, data_columns=['A', 'B'])       5434            store.append('df_tz', df, data_columns=['A', 'B'])
5434            result = store['df_tz']                                  5435            result = store['df_tz']
5435            self._compare_with_tz(result, df)                        5436            self._compare_with_tz(result, df)
5436            assert_frame_equal(result, df)                           5437            assert_frame_equal(result, df)
5437                                                                     5438
5438            # can't append with diff timezone                        5439            # can't append with diff timezone
5439            df = DataFrame(dict(A=Timestamp('20130102',              5440            df = DataFrame(dict(A=Timestamp('20130102',
5440                                            tz=gettz('US/Eastern')), 5441                                            tz=gettz('US/Eastern')),
5441                                B=Timestamp('20130102', tz=gettz('CE5442                                B=Timestamp('20130102', tz=gettz('CE
                                                                  T'))),                                                                  T'))),
5442                           index=range(5))                           5443                           index=range(5))
5443            pytest.raises(ValueError, store.append, 'df_tz', df)     5444            pytest.raises(ValueError, store.append, 'df_tz', df)
5444                                                                     5445
5445        # as index                                                   5446        # as index
5446        with ensure_clean_store(self.path) as store:                 5447        with ensure_clean_store(self.path) as store:
5447                                                                     5448
5448            # GH 4098 example                                        5449            # GH 4098 example
5449            df = DataFrame(dict(A=Series(lrange(3), index=date_range(5450            df = DataFrame(dict(A=Series(lrange(3), index=date_range(
5450                '2000-1-1', periods=3, freq='H', tz=gettz('US/Easter5451                '2000-1-1', periods=3, freq='H', tz=gettz('US/Easter
                                                                 n')))))                                                                 n')))))
5451                                                                     5452
5452            _maybe_remove(store, 'df')                               5453            _maybe_remove(store, 'df')
5453            store.put('df', df)                                      5454            store.put('df', df)
5454            result = store.select('df')                              5455            result = store.select('df')
5455            assert_frame_equal(result, df)                           5456            assert_frame_equal(result, df)
5456                                                                     5457
5457            _maybe_remove(store, 'df')                               5458            _maybe_remove(store, 'df')
5458            store.append('df', df)                                   5459            store.append('df', df)
5459            result = store.select('df')                              5460            result = store.select('df')
5460            assert_frame_equal(result, df)                           5461            assert_frame_equal(result, df)
5461                                                                     5462
5462    def test_append_with_timezones_pytz(self):                       5463    def test_append_with_timezones_pytz(self):
5463                                                                     5464
5464        from datetime import timedelta                               5465        from datetime import timedelta
5465                                                                     5466
5466        # as columns                                                 5467        # as columns
5467        with ensure_clean_store(self.path) as store:                 5468        with ensure_clean_store(self.path) as store:
5468                                                                     5469
5469            _maybe_remove(store, 'df_tz')                            5470            _maybe_remove(store, 'df_tz')
5470            df = DataFrame(dict(A=[Timestamp('20130102 2:00:00',     5471            df = DataFrame(dict(A=[Timestamp('20130102 2:00:00',
5471                                             tz='US/Eastern') +      5472                                             tz='US/Eastern') +
5472                                   timedelta(hours=1) * i            5473                                   timedelta(hours=1) * i
5473                                   for i in range(5)]))              5474                                   for i in range(5)]))
5474            store.append('df_tz', df, data_columns=['A'])            5475            store.append('df_tz', df, data_columns=['A'])
5475            result = store['df_tz']                                  5476            result = store['df_tz']
5476            self._compare_with_tz(result, df)                        5477            self._compare_with_tz(result, df)
5477            assert_frame_equal(result, df)                           5478            assert_frame_equal(result, df)
5478                                                                     5479
5479            # select with tz aware                                   5480            # select with tz aware
5480            self._compare_with_tz(store.select(                      5481            self._compare_with_tz(store.select(
5481                'df_tz', where='A>=df.A[3]'), df[df.A >= df.A[3]])   5482                'df_tz', where='A>=df.A[3]'), df[df.A >= df.A[3]])
5482                                                                     5483
5483            _maybe_remove(store, 'df_tz')                            5484            _maybe_remove(store, 'df_tz')
5484            # ensure we include dates in DST and STD time here.      5485            # ensure we include dates in DST and STD time here.
5485            df = DataFrame(dict(A=Timestamp('20130102', tz='US/Easte5486            df = DataFrame(dict(A=Timestamp('20130102', tz='US/Easte
                                                                   rn'),                                                                   rn'),
5486                                B=Timestamp('20130603', tz='US/Easte5487                                B=Timestamp('20130603', tz='US/Easte
                                                                  rn')),                                                                  rn')),
5487                           index=range(5))                           5488                           index=range(5))
5488            store.append('df_tz', df)                                5489            store.append('df_tz', df)
5489            result = store['df_tz']                                  5490            result = store['df_tz']
5490            self._compare_with_tz(result, df)                        5491            self._compare_with_tz(result, df)
5491            assert_frame_equal(result, df)                           5492            assert_frame_equal(result, df)
5492                                                                     5493
5493            df = DataFrame(dict(A=Timestamp('20130102', tz='US/Easte5494            df = DataFrame(dict(A=Timestamp('20130102', tz='US/Easte
                                                                   rn'),                                                                   rn'),
5494                                B=Timestamp('20130102', tz='EET')),  5495                                B=Timestamp('20130102', tz='EET')),
5495                           index=range(5))                           5496                           index=range(5))
5496            pytest.raises(ValueError, store.append, 'df_tz', df)     5497            pytest.raises(ValueError, store.append, 'df_tz', df)
5497                                                                     5498
5498            # this is ok                                             5499            # this is ok
5499            _maybe_remove(store, 'df_tz')                            5500            _maybe_remove(store, 'df_tz')
5500            store.append('df_tz', df, data_columns=['A', 'B'])       5501            store.append('df_tz', df, data_columns=['A', 'B'])
5501            result = store['df_tz']                                  5502            result = store['df_tz']
5502            self._compare_with_tz(result, df)                        5503            self._compare_with_tz(result, df)
5503            assert_frame_equal(result, df)                           5504            assert_frame_equal(result, df)
5504                                                                     5505
5505            # can't append with diff timezone                        5506            # can't append with diff timezone
5506            df = DataFrame(dict(A=Timestamp('20130102', tz='US/Easte5507            df = DataFrame(dict(A=Timestamp('20130102', tz='US/Easte
                                                                   rn'),                                                                   rn'),
5507                                B=Timestamp('20130102', tz='CET')),  5508                                B=Timestamp('20130102', tz='CET')),
5508                           index=range(5))                           5509                           index=range(5))
5509            pytest.raises(ValueError, store.append, 'df_tz', df)     5510            pytest.raises(ValueError, store.append, 'df_tz', df)
5510                                                                     5511
5511        # as index                                                   5512        # as index
5512        with ensure_clean_store(self.path) as store:                 5513        with ensure_clean_store(self.path) as store:
5513                                                                     5514
5514            # GH 4098 example                                        5515            # GH 4098 example
5515            df = DataFrame(dict(A=Series(lrange(3), index=date_range(5516            df = DataFrame(dict(A=Series(lrange(3), index=date_range(
5516                '2000-1-1', periods=3, freq='H', tz='US/Eastern')))) 5517                '2000-1-1', periods=3, freq='H', tz='US/Eastern'))))
5517                                                                     5518
5518            _maybe_remove(store, 'df')                               5519            _maybe_remove(store, 'df')
5519            store.put('df', df)                                      5520            store.put('df', df)
5520            result = store.select('df')                              5521            result = store.select('df')
5521            assert_frame_equal(result, df)                           5522            assert_frame_equal(result, df)
5522                                                                     5523
5523            _maybe_remove(store, 'df')                               5524            _maybe_remove(store, 'df')
5524            store.append('df', df)                                   5525            store.append('df', df)
5525            result = store.select('df')                              5526            result = store.select('df')
5526            assert_frame_equal(result, df)                           5527            assert_frame_equal(result, df)
5527                                                                     5528
5528    def test_tseries_select_index_column(self):                      5529    def test_tseries_select_index_column(self):
5529        # GH7777                                                     5530        # GH7777
5530        # selecting a UTC datetimeindex column did                   5531        # selecting a UTC datetimeindex column did
5531        # not preserve UTC tzinfo set before storing                 5532        # not preserve UTC tzinfo set before storing
5532                                                                     5533
5533        # check that no tz still works                               5534        # check that no tz still works
5534        rng = date_range('1/1/2000', '1/30/2000')                    5535        rng = date_range('1/1/2000', '1/30/2000')
5535        frame = DataFrame(np.random.randn(len(rng), 4), index=rng)   5536        frame = DataFrame(np.random.randn(len(rng), 4), index=rng)
5536                                                                     5537
5537        with ensure_clean_store(self.path) as store:                 5538        with ensure_clean_store(self.path) as store:
5538            store.append('frame', frame)                             5539            store.append('frame', frame)
5539            result = store.select_column('frame', 'index')           5540            result = store.select_column('frame', 'index')
5540            assert rng.tz == DatetimeIndex(result.values).tz         5541            assert rng.tz == DatetimeIndex(result.values).tz
5541                                                                     5542
5542        # check utc                                                  5543        # check utc
5543        rng = date_range('1/1/2000', '1/30/2000', tz='UTC')          5544        rng = date_range('1/1/2000', '1/30/2000', tz='UTC')
5544        frame = DataFrame(np.random.randn(len(rng), 4), index=rng)   5545        frame = DataFrame(np.random.randn(len(rng), 4), index=rng)
5545                                                                     5546
5546        with ensure_clean_store(self.path) as store:                 5547        with ensure_clean_store(self.path) as store:
5547            store.append('frame', frame)                             5548            store.append('frame', frame)
5548            result = store.select_column('frame', 'index')           5549            result = store.select_column('frame', 'index')
5549            assert rng.tz == result.dt.tz                            5550            assert rng.tz == result.dt.tz
5550                                                                     5551
5551        # double check non-utc                                       5552        # double check non-utc
5552        rng = date_range('1/1/2000', '1/30/2000', tz='US/Eastern')   5553        rng = date_range('1/1/2000', '1/30/2000', tz='US/Eastern')
5553        frame = DataFrame(np.random.randn(len(rng), 4), index=rng)   5554        frame = DataFrame(np.random.randn(len(rng), 4), index=rng)
5554                                                                     5555
5555        with ensure_clean_store(self.path) as store:                 5556        with ensure_clean_store(self.path) as store:
5556            store.append('frame', frame)                             5557            store.append('frame', frame)
5557            result = store.select_column('frame', 'index')           5558            result = store.select_column('frame', 'index')
5558            assert rng.tz == result.dt.tz                            5559            assert rng.tz == result.dt.tz
5559                                                                     5560
5560    def test_timezones_fixed(self):                                  5561    def test_timezones_fixed(self):
5561        with ensure_clean_store(self.path) as store:                 5562        with ensure_clean_store(self.path) as store:
5562                                                                     5563
5563            # index                                                  5564            # index
5564            rng = date_range('1/1/2000', '1/30/2000', tz='US/Eastern5565            rng = date_range('1/1/2000', '1/30/2000', tz='US/Eastern
                                                                      ')                                                                      ')
5565            df = DataFrame(np.random.randn(len(rng), 4), index=rng)  5566            df = DataFrame(np.random.randn(len(rng), 4), index=rng)
5566            store['df'] = df                                         5567            store['df'] = df
5567            result = store['df']                                     5568            result = store['df']
5568            assert_frame_equal(result, df)                           5569            assert_frame_equal(result, df)
5569                                                                     5570
5570            # as data                                                5571            # as data
5571            # GH11411                                                5572            # GH11411
5572            _maybe_remove(store, 'df')                               5573            _maybe_remove(store, 'df')
5573            df = DataFrame({'A': rng,                                5574            df = DataFrame({'A': rng,
5574                            'B': rng.tz_convert('UTC').tz_localize(N5575                            'B': rng.tz_convert('UTC').tz_localize(N
                                                                   one),                                                                   one),
5575                            'C': rng.tz_convert('CET'),              5576                            'C': rng.tz_convert('CET'),
5576                            'D': range(len(rng))}, index=rng)        5577                            'D': range(len(rng))}, index=rng)
5577            store['df'] = df                                         5578            store['df'] = df
5578            result = store['df']                                     5579            result = store['df']
5579            assert_frame_equal(result, df)                           5580            assert_frame_equal(result, df)
5580                                                                     5581
5581    def test_fixed_offset_tz(self):                                  5582    def test_fixed_offset_tz(self):
5582        rng = date_range('1/1/2000 00:00:00-07:00', '1/30/2000 00:005583        rng = date_range('1/1/2000 00:00:00-07:00', '1/30/2000 00:00
                                                             :00-07:00')                                                             :00-07:00')
5583        frame = DataFrame(np.random.randn(len(rng), 4), index=rng)   5584        frame = DataFrame(np.random.randn(len(rng), 4), index=rng)
5584                                                                     5585
5585        with ensure_clean_store(self.path) as store:                 5586        with ensure_clean_store(self.path) as store:
5586            store['frame'] = frame                                   5587            store['frame'] = frame
5587            recons = store['frame']                                  5588            recons = store['frame']
5588            tm.assert_index_equal(recons.index, rng)                 5589            tm.assert_index_equal(recons.index, rng)
5589            assert rng.tz == recons.index.tz                         5590            assert rng.tz == recons.index.tz
5590                                                                     5591
5591    def test_store_timezone(self):                                   5592    def test_store_timezone(self):
5592        # GH2852                                                     5593        # GH2852
5593        # issue storing datetime.date with a timezone as it resets w5594        # issue storing datetime.date with a timezone as it resets w
                                                                hen read                                                                hen read
5594        # back in a new timezone                                     5595        # back in a new timezone
5595                                                                     5596
5596        # original method                                            5597        # original method
5597        with ensure_clean_store(self.path) as store:                 5598        with ensure_clean_store(self.path) as store:
5598                                                                     5599
5599            today = datetime.date(2013, 9, 10)                       5600            today = datetime.date(2013, 9, 10)
5600            df = DataFrame([1, 2, 3], index=[today, today, today])   5601            df = DataFrame([1, 2, 3], index=[today, today, today])
5601            store['obj1'] = df                                       5602            store['obj1'] = df
5602            result = store['obj1']                                   5603            result = store['obj1']
5603            assert_frame_equal(result, df)                           5604            assert_frame_equal(result, df)
5604                                                                     5605
5605        # with tz setting                                            5606        # with tz setting
5606        with ensure_clean_store(self.path) as store:                 5607        with ensure_clean_store(self.path) as store:
5607                                                                     5608
5608            with set_timezone('EST5EDT'):                            5609            with set_timezone('EST5EDT'):
5609                today = datetime.date(2013, 9, 10)                   5610                today = datetime.date(2013, 9, 10)
5610                df = DataFrame([1, 2, 3], index=[today, today, today5611                df = DataFrame([1, 2, 3], index=[today, today, today
                                                                      ])                                                                      ])
5611                store['obj1'] = df                                   5612                store['obj1'] = df
5612                                                                     5613
5613            with set_timezone('CST6CDT'):                            5614            with set_timezone('CST6CDT'):
5614                result = store['obj1']                               5615                result = store['obj1']
5615                                                                     5616
5616            assert_frame_equal(result, df)                           5617            assert_frame_equal(result, df)
5617                                                                     5618
5618    def test_legacy_datetimetz_object(self):                         5619    def test_legacy_datetimetz_object(self):
5619        # legacy from < 0.17.0                                       5620        # legacy from < 0.17.0
5620        # 8260                                                       5621        # 8260
5621        expected = DataFrame(dict(A=Timestamp('20130102', tz='US/Eas5622        expected = DataFrame(dict(A=Timestamp('20130102', tz='US/Eas
                                                                 tern'),                                                                 tern'),
5622                                  B=Timestamp('20130603', tz='CET')),5623                                  B=Timestamp('20130603', tz='CET')),
5623                             index=range(5))                         5624                             index=range(5))
5624        with ensure_clean_store(                                     5625        with ensure_clean_store(
5625                tm.get_data_path('legacy_hdf/datetimetz_object.h5'), 5626                tm.get_data_path('legacy_hdf/datetimetz_object.h5'),
5626                mode='r') as store:                                  5627                mode='r') as store:
5627            result = store['df']                                     5628            result = store['df']
5628            assert_frame_equal(result, expected)                     5629            assert_frame_equal(result, expected)
5629                                                                     5630
5630    def test_dst_transitions(self):                                  5631    def test_dst_transitions(self):
5631        # make sure we are not failing on transaitions               5632        # make sure we are not failing on transaitions
5632        with ensure_clean_store(self.path) as store:                 5633        with ensure_clean_store(self.path) as store:
5633            times = pd.date_range("2013-10-26 23:00", "2013-10-27 015634            times = pd.date_range("2013-10-26 23:00", "2013-10-27 01
                                                                   :00",                                                                   :00",
5634                                  tz="Europe/London",                5635                                  tz="Europe/London",
5635                                  freq="H",                          5636                                  freq="H",
5636                                  ambiguous='infer')                 5637                                  ambiguous='infer')
5637                                                                     5638
5638            for i in [times, times + pd.Timedelta('10min')]:         5639            for i in [times, times + pd.Timedelta('10min')]:
5639                _maybe_remove(store, 'df')                           5640                _maybe_remove(store, 'df')
5640                df = DataFrame({'A': range(len(i)), 'B': i}, index=i)5641                df = DataFrame({'A': range(len(i)), 'B': i}, index=i)
5641                store.append('df', df)                               5642                store.append('df', df)
5642                result = store.select('df')                          5643                result = store.select('df')
5643                assert_frame_equal(result, df)                       5644                assert_frame_equal(result, df)